ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-21 02:01:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 2916.grid-manager-integration-tests.2

Browse Source
This commit is contained in:
Jean-Paul Calderone 2023-03-16 08:33:16 -04:00
commit 35525bc10e
165 changed files with 23357 additions and 2931 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
.circleci/circleci.txt Normal file
View File

@ -0,0 +1,78 @@
# A master build looks like this:
# BASH_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=master
# CIRCLE_BUILD_NUM=76545
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76545
# CIRCLE_JOB=NixOS 21.11
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=ed0bda2d7456f4a2cd60870072e1fe79864a49a1
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build
# CIRCLE_USERNAME=alice
# CIRCLE_WORKFLOW_ID=6d9bb71c-be3a-4659-bf27-60954180619b
# CIRCLE_WORKFLOW_JOB_ID=0793c975-7b9f-489f-909b-8349b72d2785
# CIRCLE_WORKFLOW_WORKSPACE_ID=6d9bb71c-be3a-4659-bf27-60954180619b
# CIRCLE_WORKING_DIRECTORY=~/project
# A build of an in-repo PR looks like this:
# BASH_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=3946-less-chatty-downloads
# CIRCLE_BUILD_NUM=76612
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76612
# CIRCLE_JOB=NixOS 21.11
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=921a2083dcefdb5f431cdac195fc9ac510605349
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build
# CIRCLE_USERNAME=bob
# CIRCLE_WORKFLOW_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1
# CIRCLE_WORKFLOW_JOB_ID=316ca408-81b4-4c96-bbdd-644e4c3e01e5
# CIRCLE_WORKFLOW_WORKSPACE_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1
# CIRCLE_WORKING_DIRECTORY=~/project
# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# A build of a PR from a fork looks like this:
# BASH_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=pull/1252
# CIRCLE_BUILD_NUM=76678
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76678
# CIRCLE_JOB=NixOS 21.05
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_PR_NUMBER=1252
# CIRCLE_PR_REPONAME=tahoe-lafs
# CIRCLE_PR_USERNAME=carol
# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=15c7916e0812e6baa2a931cd54b18f3382a8456e
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build
# CIRCLE_USERNAME=
# CIRCLE_WORKFLOW_ID=19c917c8-3a38-4b20-ac10-3265259fa03e
# CIRCLE_WORKFLOW_JOB_ID=58e95215-eccf-4664-a231-1dba7fd2d323
# CIRCLE_WORKFLOW_WORKSPACE_ID=19c917c8-3a38-4b20-ac10-3265259fa03e
# CIRCLE_WORKING_DIRECTORY=~/project
# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# A build of a PR from a fork where the owner has enabled CircleCI looks
# the same as a build of an in-repo PR, except it runs on th owner's
# CircleCI namespace.

249
.circleci/config.yml
View File

@ -11,32 +11,72 @@
#
version: 2.1
# A template that can be shared between the two different image-building
# workflows.
.images: &IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub credentials
# available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-11": &DOCKERHUB_CONTEXT
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
parameters:
# Control whether the image-building workflow runs as part of this pipeline.
# Generally we do not want this to run because we don't need our
# dependencies to move around all the time and because building the image
# takes a couple minutes.
#
# An easy way to trigger a pipeline with this set to true is with the
# rebuild-images.sh tool in this directory. You can also do so via the
# CircleCI web UI.
build-images:
default: false
type: "boolean"
# Control whether the test-running workflow runs as part of this pipeline.
# Generally we do want this to run because running the tests is the primary
# purpose of this pipeline.
run-tests:
default: true
type: "boolean"
workflows:
ci:
when: "<< pipeline.parameters.run-tests >>"
jobs:
# Start with jobs testing various platforms.
- "debian-10":
{}
- "debian-11":
{}
- "ubuntu-20-04":
{}
- "ubuntu-18-04":
requires:
- "ubuntu-20-04"
# Equivalent to RHEL 8; CentOS 8 is dead.
- "oraclelinux-8":
{}
- "nixos":
name: "NixOS 21.05"
nixpkgs: "21.05"
name: "NixOS 22.11"
nixpkgs: "22.11"
- "nixos":
name: "NixOS 21.11"
nixpkgs: "21.11"
name: "NixOS unstable"
nixpkgs: "unstable"
# Eventually, test against PyPy 3.8
#- "pypy27-buster":
@ -54,6 +94,9 @@ workflows:
{}
- "integration":
# Run even the slow integration tests here. We need the `--` to
# sneak past tox and get to pytest.
tox-args: "-- --runslow integration"
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
@ -65,41 +108,10 @@ workflows:
{}
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
# faster and takes various spurious failures out of the critical path.
triggers:
# Build once a day
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- "master"
<<: *IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub
# credentials available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-10": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "build-image-debian-11":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
# Build as part of the workflow but only if requested.
when: "<< pipeline.parameters.build-images >>"
jobs:
@ -133,10 +145,10 @@ jobs:
steps:
- "checkout"
- run:
- run: &INSTALL_TOX
name: "Install tox"
command: |
pip install --user tox
pip install --user 'tox~=3.0'
- run:
name: "Static-ish code checks"
@ -152,9 +164,7 @@ jobs:
- "checkout"
- run:
name: "Install tox"
command: |
pip install --user tox
<<: *INSTALL_TOX
- run:
name: "Make PyInstaller executable"
@ -169,12 +179,7 @@ jobs:
command: |
dist/Tahoe-LAFS/tahoe --version
debian-10: &DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:10-py3.7"
user: "nobody"
debian-11: &DEBIAN
environment: &UTF_8_ENVIRONMENT
# In general, the test suite is not allowed to fail while the job
# succeeds. But you can set this to "yes" if you want it to be
@ -186,7 +191,7 @@ jobs:
# filenames and argv).
LANG: "en_US.UTF-8"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
@ -254,15 +259,11 @@ jobs:
/tmp/venv/bin/codecov
fi
debian-11:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Restore later using PyPy3.8
# pypy27-buster:
@ -296,6 +297,14 @@ jobs:
integration:
<<: *DEBIAN
parameters:
tox-args:
description: >-
Additional arguments to pass to the tox command.
type: "string"
default: ""
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
@ -308,28 +317,15 @@ jobs:
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
# Pass on anything we got in our parameters.
TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >>"
steps:
- "checkout"
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
ubuntu-18-04: &UBUNTU_18_04
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# The default trial args include --rterrors which is incompatible with
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
ubuntu-20-04:
<<: *DEBIAN
docker:
@ -382,7 +378,7 @@ jobs:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.3.16"
image: "nixos/nix:2.10.3"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
@ -392,27 +388,21 @@ jobs:
steps:
- "run":
# The nixos/nix image does not include ssh. Install it so the
# `checkout` step will succeed. We also want cachix for
# Nix-friendly caching.
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
nix-env \
--file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
--file $NIXPKGS \
--install \
-A openssh cachix bash
-A cachix bash
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"
- run:
name: "Cachix setup"
# Record the store paths that exist before we did much. There's no
# reason to cache these, they're either in the image or have to be
# retrieved before we can use cachix to restore from cache.
command: |
cachix use "${CACHIX_NAME}"
nix path-info --all > /tmp/store-path-pre-build
- "run":
# The Nix package doesn't know how to do this part, unfortunately.
name: "Generate version"
@ -422,67 +412,23 @@ jobs:
--run 'python setup.py update_version'
- "run":
name: "Build"
name: "Test"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suites the build environment we're paying
# for (the free one!).
#
# Also, let it run more than one job at a time because we have to
# build a couple simple little dependencies that don't take
# advantage of multiple cores and we get a little speedup by doing
# them in parallel.
nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
- "run":
name: "Test"
command: |
# Let it go somewhat wild for the test suite itself
nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
- run:
# Send any new store objects to cachix.
name: "Push to Cachix"
when: "always"
command: |
# Cribbed from
# https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
if [ -n "$CIRCLE_PR_NUMBER" ]; then
# I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
# number of the PR being built". Sorry, dear reader, you have
# guessed poorly. It is also conditionally set based on whether
# this is a PR from a fork or not.
#
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
echo "Skipping Cachix push for forked PR."
else
# If this *isn't* a build from a fork then we have the Cachix
# write key in our environment and we can push any new objects
# to Cachix.
#
# To decide what to push, we inspect the list of store objects
# that existed before and after we did most of our work. Any
# that are new after the work is probably a useful thing to have
# around so push it to the cache. We exclude all derivation
# objects (.drv files) because they're cheap to reconstruct and
# by the time you know their cache key you've already done all
# the work anyway.
#
# This shell expression for finding the objects and pushing them
# was from the Cachix docs:
#
# https://docs.cachix.org/continuous-integration-setup/circleci.html
#
# but they seem to have removed it now.
bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
fi
source .circleci/lib.sh
cache_if_able nix-build \
--cores 8 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
nix/tests.nix
typechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -494,7 +440,7 @@ jobs:
docs:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -545,15 +491,6 @@ jobs:
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
build-image-debian-10:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "10"
PYTHON_VERSION: "3.7"
build-image-debian-11:
<<: *BUILD_IMAGE
@ -562,14 +499,6 @@ jobs:
TAG: "11"
PYTHON_VERSION: "3.9"
build-image-ubuntu-18-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
PYTHON_VERSION: "3.7"
build-image-ubuntu-20-04:
<<: *BUILD_IMAGE

119
.circleci/lib.sh Normal file
View File

@ -0,0 +1,119 @@
# Run a command, enabling cache writes to cachix if possible. The command is
# accepted as a variable number of positional arguments (like argv).
function cache_if_able() {
# Dump some info about our build environment.
describe_build
if is_cache_writeable; then
# If the cache is available we'll use it. This lets fork owners set
# up their own caching if they want.
echo "Cachix credentials present; will attempt to write to cache."
# The `cachix watch-exec ...` does our cache population. When it sees
# something added to the store (I guess) it pushes it to the named
# cache.
cachix watch-exec "${CACHIX_NAME}" -- "$@"
else
if is_cache_required; then
echo "Required credentials (CACHIX_AUTH_TOKEN) are missing."
return 1
else
echo "Cachix credentials missing; will not attempt cache writes."
"$@"
fi
fi
}
function is_cache_writeable() {
# We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo
# jobs will get this from CircleCI configuration but jobs from forks may
# not.
[ -v CACHIX_AUTH_TOKEN ]
}
function is_cache_required() {
# If we're building in tahoe-lafs/tahoe-lafs then we must use the cache.
# If we're building anything from a fork then we're allowed to not have
# the credentials.
is_upstream
}
# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs
# repository itself (and so we expect to have cache credentials available),
# failure otherwise.
#
# See circleci.txt for notes about how this determination is made.
function is_upstream() {
# CIRCLE_PROJECT_USERNAME is set to the org the build is happening for.
# If a PR targets a fork of the repo then this is set to something other
# than "tahoe-lafs".
[ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] &&
# CIRCLE_BRANCH is set to the real branch name for in-repo PRs and
# "pull/NNNN" for pull requests from forks.
#
# CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full
# URLs of the PR pages which share an underlying branch, with one of
# them ended with that same "pull/NNNN" for PRs from forks.
! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS"
}
# Return success if splitting $3 on $2 results in an array with any element
# that ends with $1, failure otherwise.
function any_element_endswith() {
suffix=$1
shift
sep=$1
shift
haystack=$1
shift
IFS="${sep}" read -r -a elements <<< "$haystack"
for elem in "${elements[@]}"; do
if endswith "$suffix" "$elem"; then
return 0
fi
done
return 1
}
# Return success if $2 ends with $1, failure otherwise.
function endswith() {
suffix=$1
shift
haystack=$1
shift
case "$haystack" in
*${suffix})
return 0
;;
*)
return 1
;;
esac
}
function describe_build() {
echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}"
echo "Building branch: ${CIRCLE_BRANCH}"
if is_upstream; then
echo "Upstream build."
else
echo "Non-upstream build."
fi
if is_cache_required; then
echo "Cache is required."
else
echo "Cache not required."
fi
if is_cache_writeable; then
echo "Cache is writeable."
else
echo "Cache not writeable."
fi
}

2
.circleci/populate-wheelhouse.sh
View File

@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel"
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
TEST_DEPS="tox~=3.0 codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.

20
.circleci/rebuild-images.sh Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
# Get your API token here:
# https://app.circleci.com/settings/user/tokens
API_TOKEN=$1
shift
# Name the branch you want to trigger the build for
BRANCH=$1
shift
curl \
--verbose \
--request POST \
--url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \
--header "Circle-Token: $API_TOKEN" \
--header "content-type: application/json" \
--data '{"branch":"'"$BRANCH"'","parameters":{"build-images":true,"run-tests":false}}'

9
.circleci/run-tests.sh
View File

@ -45,14 +45,15 @@ fi
# A prefix for the test command that ensure it will exit after no more than a
# certain amount of time. Ideally, we would only enforce a "silent" period
# timeout but there isn't obviously a ready-made tool for that. The test
# suite only takes about 5 - 6 minutes on CircleCI right now. 15 minutes
# seems like a moderately safe window.
# timeout but there isn't obviously a ready-made tool for that. The unit test
# suite only takes about 5 - 6 minutes on CircleCI right now. The integration
# tests are a bit longer than that. 45 minutes seems like a moderately safe
# window.
#
# This is primarily aimed at catching hangs on the PyPy job which runs for
# about 21 minutes and then gets killed by CircleCI in a way that fails the
# job and bypasses our "allowed failure" logic.
TIMEOUT="timeout --kill-after 1m 15m"
TIMEOUT="timeout --kill-after 1m 45m"
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable

145
.github/workflows/ci.yml vendored
View File

@ -6,6 +6,16 @@ on:
- "master"
pull_request:
# At the start of each workflow run, GitHub creates a unique
# GITHUB_TOKEN secret to use in the workflow. It is a good idea for
# this GITHUB_TOKEN to have the minimum of permissions. See:
#
# - https://docs.github.com/en/actions/security-guides/automatic-token-authentication
# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
#
permissions:
contents: read
# Control to what degree jobs in this workflow will run concurrently with
# other instances of themselves.
#
@ -38,73 +48,67 @@ jobs:
- windows-latest
- ubuntu-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
- "3.11"
include:
# On macOS don't bother with 3.7-3.8, just to get faster builds.
# On macOS don't bother with 3.8, just to get faster builds.
- os: macos-latest
python-version: "3.9"
- os: macos-latest
python-version: "3.10"
python-version: "3.11"
# We only support PyPy on Linux at the moment.
- os: ubuntu-latest
python-version: "pypy-3.7"
- os: ubuntu-latest
python-version: "pypy-3.8"
- os: ubuntu-latest
python-version: "pypy-3.9"
steps:
# See https://github.com/actions/checkout. A fetch-depth of 0
# fetches all tags and branches.
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
# To use pip caching with GitHub Actions in an OS-independent
# manner, we need `pip cache dir` command, which became
# available since pip v20.1+. At the time of writing this,
# GitHub Actions offers pip v20.3.3 for both ubuntu-latest and
# windows-latest, and pip v20.3.1 for macos-latest.
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
# See https://github.com/actions/cache
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade codecov tox tox-gh-actions setuptools
pip install --upgrade codecov "tox<4" tox-gh-actions setuptools
pip list
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run tox for corresponding Python version
if: ${{ !contains(matrix.os, 'windows') }}
run: python -m tox
# On Windows, a non-blocking pipe might respond (when emulating Unix-y
# API) with ENOSPC to indicate buffer full. Trial doesn't handle this
# well, so it breaks test runs. To attempt to solve this, we pipe the
# output through passthrough.py that will hopefully be able to do the right
# thing by using Windows APIs.
- name: Run tox for corresponding Python version
if: ${{ contains(matrix.os, 'windows') }}
run: |
pip install twisted pywin32
python -m tox | python misc/windows-enospc/passthrough.py
- name: Upload eliot.log
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: eliot.log
path: eliot.log
- name: Upload trial log
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: test.log
path: _trial_temp/test.log
@ -161,21 +165,22 @@ jobs:
strategy:
fail-fast: false
matrix:
os:
- windows-latest
- ubuntu-latest
python-version:
- 3.7
- 3.9
include:
# On macOS don't bother with 3.7, just to get faster builds.
- os: macos-latest
python-version: 3.9
python-version: "3.9"
force-foolscap: false
- os: windows-latest
python-version: "3.9"
force-foolscap: false
# 22.04 has some issue with Tor at the moment:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
- os: ubuntu-20.04
python-version: "3.11"
force-foolscap: false
steps:
- name: Install Tor [Ubuntu]
if: matrix.os == 'ubuntu-latest'
if: ${{ contains(matrix.os, 'ubuntu') }}
run: sudo apt install tor
# TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744.
@ -188,51 +193,51 @@ jobs:
- name: Install Tor [Windows]
if: matrix.os == 'windows-latest'
uses: crazy-max/ghaction-chocolatey@v1
uses: crazy-max/ghaction-chocolatey@v2
with:
args: install tor
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade tox
pip install --upgrade "tox<4"
pip list
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run "Python 3 integration tests"
if: "${{ !matrix.force-foolscap }}"
env:
# On macOS this is necessary to ensure unix socket paths for tor
# aren't too long. On Windows tox won't pass it through so it has no
# effect. On Linux it doesn't make a difference one way or another.
TMPDIR: "/tmp"
run: tox -e integration
run: |
tox -e integration
- name: Run "Python 3 integration tests (force Foolscap)"
if: "${{ matrix.force-foolscap }}"
env:
# On macOS this is necessary to ensure unix socket paths for tor
# aren't too long. On Windows tox won't pass it through so it has no
# effect. On Linux it doesn't make a difference one way or another.
TMPDIR: "/tmp"
run: |
tox -e integration -- --force-foolscap integration/
- name: Upload eliot.log in case of failure
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
if: failure()
with:
name: integration.eliot.json
@ -253,31 +258,19 @@ jobs:
steps:
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade tox
pip install --upgrade "tox<4"
pip list
- name: Display tool versions
@ -291,7 +284,7 @@ jobs:
run: dist/Tahoe-LAFS/tahoe --version
- name: Upload PyInstaller package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: Tahoe-LAFS-${{ matrix.os }}-Python-${{ matrix.python-version }}
path: dist/Tahoe-LAFS-*-*.*

10
Dockerfile
View File

@ -1,10 +0,0 @@
FROM python:2.7
ADD . /tahoe-lafs
RUN \
cd /tahoe-lafs && \
git pull --depth=100 && \
pip install . && \
rm -rf ~/.cache/
WORKDIR /root

25
Dockerfile.dev
View File

@ -1,25 +0,0 @@
FROM debian:9
LABEL maintainer "gordon@leastauthority.com"
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get -yq upgrade
RUN DEBIAN_FRONTEND=noninteractive apt-get -yq install build-essential python-dev libffi-dev libssl-dev python-virtualenv git
RUN \
git clone https://github.com/tahoe-lafs/tahoe-lafs.git /root/tahoe-lafs; \
cd /root/tahoe-lafs; \
virtualenv --python=python2.7 venv; \
./venv/bin/pip install --upgrade setuptools; \
./venv/bin/pip install --editable .; \
./venv/bin/tahoe --version;
RUN \
cd /root; \
mkdir /root/.tahoe-client; \
mkdir /root/.tahoe-introducer; \
mkdir /root/.tahoe-server;
RUN /root/tahoe-lafs/venv/bin/tahoe create-introducer --location=tcp:introducer:3458 --port=tcp:3458 /root/.tahoe-introducer
RUN /root/tahoe-lafs/venv/bin/tahoe start /root/.tahoe-introducer
RUN /root/tahoe-lafs/venv/bin/tahoe create-node --location=tcp:server:3457 --port=tcp:3457 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) /root/.tahoe-server
RUN /root/tahoe-lafs/venv/bin/tahoe create-client --webport=3456 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) --basedir=/root/.tahoe-client --shares-needed=1 --shares-happy=1 --shares-total=1
VOLUME ["/root/.tahoe-client", "/root/.tahoe-server", "/root/.tahoe-introducer"]
EXPOSE 3456 3457 3458
ENTRYPOINT ["/root/tahoe-lafs/venv/bin/tahoe"]
CMD []

59
Makefile
View File

@ -224,3 +224,62 @@ src/allmydata/_version.py:
.tox/create-venvs.log: tox.ini setup.py
tox --notest -p all | tee -a "$(@)"
# to make a new release:
# - create a ticket for the release in Trac
# - ensure local copy is up-to-date
# - create a branch like "XXXX.release" from up-to-date master
# - in the branch, run "make release"
# - run "make release-test"
# - perform any other sanity-checks on the release
# - run "make release-upload"
# Note that several commands below hard-code "meejah"; if you are
# someone else please adjust them.
release:
@echo "Is checkout clean?"
git diff-files --quiet
git diff-index --quiet --cached HEAD --
@echo "Clean docs build area"
rm -rf docs/_build/
@echo "Install required build software"
python3 -m pip install --editable .[build]
@echo "Test README"
python3 setup.py check -r -s
@echo "Update NEWS"
python3 -m towncrier build --yes --version `python3 misc/build_helpers/update-version.py --no-tag`
git add -u
git commit -m "update NEWS for release"
# note that this always bumps the "middle" number, e.g. from 1.17.1 -> 1.18.0
# and produces a tag into the Git repository
@echo "Bump version and create tag"
python3 misc/build_helpers/update-version.py
@echo "Build and sign wheel"
python3 setup.py bdist_wheel
gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
ls dist/*`git describe | cut -b 12-`*
@echo "Build and sign source-dist"
python3 setup.py sdist
gpg --pinentry=loopback -u meejah@meejah.ca --armor --detach-sign dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz
ls dist/*`git describe | cut -b 12-`*
# basically just a bare-minimum smoke-test that it installs and runs
release-test:
gpg --verify dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc
gpg --verify dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc
virtualenv testmf_venv
testmf_venv/bin/pip install dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl
testmf_venv/bin/tahoe --version
rm -rf testmf_venv
release-upload:
scp dist/*`git describe | cut -b 12-`* meejah@tahoe-lafs.org:/home/source/downloads
git push origin_push tahoe-lafs-`git describe | cut -b 12-`
twine upload dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl dist/tahoe_lafs-`git describe | cut -b 12-`-py3-none-any.whl.asc dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz dist/tahoe-lafs-`git describe | cut -b 12-`.tar.gz.asc

41
NEWS.rst
View File

@ -5,6 +5,47 @@ User-Visible Changes in Tahoe-LAFS
==================================
.. towncrier start line
Release 1.18.0 (2022-10-02)
'''''''''''''''''''''''''''
Backwards Incompatible Changes
------------------------------
- Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. (`#3865 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3865>`_)
- Python 3.7 or later is now required; Python 2 is no longer supported. (`#3873 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3873>`_)
- Share corruption reports stored on disk are now always encoded in UTF-8. (`#3879 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3879>`_)
- Record both the PID and the process creation-time:
a new kind of pidfile in `running.process` records both
the PID and the creation-time of the process. This facilitates
automatic discovery of a "stale" pidfile that points to a
currently-running process. If the recorded creation-time matches
the creation-time of the running process, then it is a still-running
`tahoe run` process. Otherwise, the file is stale.
The `twistd.pid` file is no longer present. (`#3926 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3926>`_)
Features
--------
- The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
Some code existed to allow tests to shorten this and it's
conceptually possible a modified client produced mutables
with different key-sizes. However, the spec says that they
must be 2048 bits. If you happen to have a capability with
a key-size different from 2048 you may use 1.17.1 or earlier
to read the content. (`#3828 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3828>`_)
- "make" based release automation (`#3846 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3846>`_)
Misc/Other
----------
- `#3327 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3327>`_, `#3526 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3526>`_, `#3697 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3697>`_, `#3709 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3709>`_, `#3786 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3786>`_, `#3788 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3788>`_, `#3802 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3802>`_, `#3816 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3816>`_, `#3855 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3855>`_, `#3858 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3858>`_, `#3859 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3859>`_, `#3860 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3860>`_, `#3867 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3867>`_, `#3868 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3868>`_, `#3871 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3871>`_, `#3872 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3872>`_, `#3875 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3875>`_, `#3876 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3876>`_, `#3877 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3877>`_, `#3881 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3881>`_, `#3882 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3882>`_, `#3883 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3883>`_, `#3889 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3889>`_, `#3890 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3890>`_, `#3891 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3891>`_, `#3893 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3893>`_, `#3895 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3895>`_, `#3896 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3896>`_, `#3898 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3898>`_, `#3900 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3900>`_, `#3909 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3909>`_, `#3913 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3913>`_, `#3915 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3915>`_, `#3916 <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3916>`_
Release 1.17.1 (2022-01-07)
'''''''''''''''''''''''''''

2
README.rst
View File

@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS <docs/running.rst>`__
🐍 Python 2
-----------
Python 3.7 or later is now required.
Python 3.8 or later is required.
If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1.

138
benchmarks/upload_download.py Normal file
View File

@ -0,0 +1,138 @@
"""
First attempt at benchmarking uploads and downloads.
To run:
$ pytest benchmarks/upload_download.py -s -v -Wignore
To add latency of e.g. 60ms on Linux:
$ tc qdisc add dev lo root netem delay 30ms
To reset:
$ tc qdisc del dev lo root netem
Frequency scaling can spoil the results.
To see the range of frequency scaling on a Linux system:
$ cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_available_frequencies
And to pin the CPU frequency to the lower bound found in these files:
$ sudo cpupower frequency-set -f <lowest available frequency>
TODO Parameterization (pytest?)
- Foolscap vs not foolscap
- Number of nodes
- Data size
- Number of needed/happy/total shares.
CAVEATS: The goal here isn't a realistic benchmark, or a benchmark that will be
measured over time, or is expected to be maintainable over time. This is just
a quick and easy way to measure the speed of certain operations, compare HTTP
and Foolscap, and see the short-term impact of changes.
Eventually this will be replaced by a real benchmark suite that can be run over
time to measure something more meaningful.
"""
from time import time, process_time
from contextlib import contextmanager
from tempfile import mkdtemp
import os
from twisted.trial.unittest import TestCase
from twisted.internet.defer import gatherResults
from allmydata.util.deferredutil import async_to_deferred
from allmydata.util.consumer import MemoryConsumer
from allmydata.test.common_system import SystemTestMixin
from allmydata.immutable.upload import Data as UData
from allmydata.mutable.publish import MutableData
@contextmanager
def timeit(name):
start = time()
start_cpu = process_time()
try:
yield
finally:
print(
f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU"
)
class ImmutableBenchmarks(SystemTestMixin, TestCase):
"""Benchmarks for immutables."""
# To use Foolscap, change to True:
FORCE_FOOLSCAP_FOR_STORAGE = False
# Don't reduce HTTP connection timeouts, that messes up the more aggressive
# benchmarks:
REDUCE_HTTP_CLIENT_TIMEOUT = False
@async_to_deferred
async def setUp(self):
SystemTestMixin.setUp(self)
self.basedir = os.path.join(mkdtemp(), "nodes")
# 2 nodes
await self.set_up_nodes(2)
# 1 share
for c in self.clients:
c.encoding_params["k"] = 1
c.encoding_params["happy"] = 1
c.encoding_params["n"] = 1
print()
@async_to_deferred
async def test_upload_and_download_immutable(self):
# To test larger files, change this:
DATA = b"Some data to upload\n" * 10
for i in range(5):
# 1. Upload:
with timeit(" upload"):
uploader = self.clients[0].getServiceNamed("uploader")
results = await uploader.upload(UData(DATA, convergence=None))
# 2. Download:
with timeit("download"):
uri = results.get_uri()
node = self.clients[1].create_node_from_uri(uri)
mc = await node.read(MemoryConsumer(), 0, None)
self.assertEqual(b"".join(mc.chunks), DATA)
@async_to_deferred
async def test_upload_and_download_mutable(self):
# To test larger files, change this:
DATA = b"Some data to upload\n" * 10
for i in range(5):
# 1. Upload:
with timeit(" upload"):
result = await self.clients[0].create_mutable_file(MutableData(DATA))
# 2. Download:
with timeit("download"):
data = await result.download_best_version()
self.assertEqual(data, DATA)
@async_to_deferred
async def test_upload_mutable_in_parallel(self):
# To test larger files, change this:
DATA = b"Some data to upload\n" * 1_000_000
with timeit(" upload"):
await gatherResults([
self.clients[0].create_mutable_file(MutableData(DATA))
for _ in range(20)
])

103
default.nix
View File

@ -1,8 +1,7 @@
let
# sources.nix contains information about which versions of some of our
# dependencies we should use. since we use it to pin nixpkgs and the PyPI
# package database, roughly all the rest of our dependencies are *also*
# pinned - indirectly.
# dependencies we should use. since we use it to pin nixpkgs, all the rest
# of our dependencies are *also* pinned - indirectly.
#
# sources.nix is managed using a tool called `niv`. as an example, to
# update to the most recent version of nixpkgs from the 21.11 maintenance
@ -10,93 +9,45 @@ let
#
# niv update nixpkgs-21.11
#
# or, to update the PyPI package database -- which is necessary to make any
# newly released packages visible -- you likewise run:
#
# niv update pypi-deps-db
#
# niv also supports chosing a specific revision, following a different
# branch, etc. find complete documentation for the tool at
# https://github.com/nmattia/niv
sources = import nix/sources.nix;
in
{
pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
# niv-managed sources data
, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
# for dependency resolution
, pythonVersion ? "python310" # a string choosing the python derivation from
# nixpkgs to target
, pythonVersion ? "python37" # a string choosing the python derivation from
# nixpkgs to target
, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
# the dependencies of which the resulting
# package will also depend on. Include all of the
# runtime extras by default because the incremental
# cost of including them is a lot smaller than the
# cost of re-building the whole thing to add them.
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
# the dependencies of which the resulting package
# will also depend on. Include all of the runtime
# extras by default because the incremental cost of
# including them is a lot smaller than the cost of
# re-building the whole thing to add them.
, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
# the tahoe-lafs package
inherit pkgs pypiData;
python = pythonVersion;
}
}:
# The project name, version, and most other metadata are automatically
# extracted from the source. Some requirements are not properly extracted
# and those cases are handled below. The version can only be extracted if
# `setup.py update_version` has been run (this is not at all ideal but it
# seems difficult to fix) - so for now just be sure to run that first.
mach-nix.buildPythonPackage rec {
# Define the location of the Tahoe-LAFS source to be packaged. Clean up all
# as many of the non-source files (eg the `.git` directory, `~` backup
# files, nix's own `result` symlink, etc) as possible to avoid needing to
# re-build when files that make no difference to the package have changed.
src = pkgs.lib.cleanSource ./.;
with (pkgs.${pythonVersion}.override {
packageOverrides = self: super: {
# Some dependencies aren't packaged in nixpkgs so supply our own packages.
pycddl = self.callPackage ./nix/pycddl.nix { };
txi2p = self.callPackage ./nix/txi2p.nix { };
};
}).pkgs;
callPackage ./nix/tahoe-lafs.nix {
# Select whichever package extras were requested.
inherit extras;
inherit extrasNames;
# Define some extra requirements that mach-nix does not automatically detect
# from inspection of the source. We typically don't need to put version
# constraints on any of these requirements. The pypi-deps-db we're
# operating with makes dependency resolution deterministic so as long as it
# works once it will always work. It could be that in the future we update
# pypi-deps-db and an incompatibility arises - in which case it would make
# sense to apply some version constraints here.
requirementsExtra = ''
# mach-nix does not yet support pyproject.toml which means it misses any
# build-time requirements of our dependencies which are declared in such a
# file. Tell it about them here.
setuptools_rust
# Define the location of the Tahoe-LAFS source to be packaged (the same
# directory as contains this file). Clean up as many of the non-source
# files (eg the `.git` directory, `~` backup files, nix's own `result`
# symlink, etc) as possible to avoid needing to re-build when files that
# make no difference to the package have changed.
tahoe-lafs-src = pkgs.lib.cleanSource ./.;
# mach-nix does not yet parse environment markers (e.g. "python > '3.0'")
# correctly. It misses all of our requirements which have an environment marker.
# Duplicate them here.
foolscap
eliot
pyrsistent
collections-extended
'';
# Specify where mach-nix should find packages for our Python dependencies.
# There are some reasonable defaults so we only need to specify certain
# packages where the default configuration runs into some issue.
providers = {
};
# Define certain overrides to the way Python dependencies are built.
_ = {
# Remove a click-default-group patch for a test suite problem which no
# longer applies because the project apparently no longer has a test suite
# in its source distribution.
click-default-group.patches = [];
};
passthru.meta.mach-nix = {
inherit providers _;
};
doCheck = false;
}

49
docker-compose.yml
View File

@ -1,49 +0,0 @@
version: '2'
services:
client:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3456:3456"
depends_on:
- "introducer"
- "server"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-client"]
server:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3457:3457"
depends_on:
- "introducer"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-server"]
introducer:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3458:3458"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-introducer"]

47
docs/check_running.py Normal file
View File

@ -0,0 +1,47 @@
import psutil
import filelock
def can_spawn_tahoe(pidfile):
"""
Determine if we can spawn a Tahoe-LAFS for the given pidfile. That
pidfile may be deleted if it is stale.
:param pathlib.Path pidfile: the file to check, that is the Path
to "running.process" in a Tahoe-LAFS configuration directory
:returns bool: True if we can spawn `tahoe run` here
"""
lockpath = pidfile.parent / (pidfile.name + ".lock")
with filelock.FileLock(lockpath):
try:
with pidfile.open("r") as f:
pid, create_time = f.read().strip().split(" ", 1)
except FileNotFoundError:
return True
# somewhat interesting: we have a pidfile
pid = int(pid)
create_time = float(create_time)
try:
proc = psutil.Process(pid)
# most interesting case: there _is_ a process running at the
# recorded PID -- but did it just happen to get that PID, or
# is it the very same one that wrote the file?
if create_time == proc.create_time():
# _not_ stale! another intance is still running against
# this configuration
return False
except psutil.NoSuchProcess:
pass
# the file is stale
pidfile.unlink()
return True
from pathlib import Path
print("can spawn?", can_spawn_tahoe(Path("running.process")))

13
docs/managed-grid.rst
View File

@ -45,7 +45,7 @@ the statement: "Grid Manager X suggests you use storage-server Y to
upload shares to" (X and Y are public-keys). Such a certificate
consists of:
- a version
- the version of the format the certificate conforms to (`1`)
- the public-key of a storage-server
- an expiry timestamp
- a signature of the above
@ -55,6 +55,10 @@ certificate, or no certificate) because clients check the ciphertext
and re-assembled plaintext against the keys in the capability;
"grid-manager" certificates only control uploads.
Clients make use of this functionality by configuring one or more Grid Manager public keys.
This tells the client to only upload to storage-servers that have a currently-valid certificate from any of the Grid Managers their client allows.
In case none are configured, the default behavior (of using any storage server) prevails.
Grid Manager Data Storage
-------------------------
@ -78,7 +82,7 @@ ever available on stdout.
The configuration is a JSON document. It is subject to change as Grid
Manager evolves. It contains a version number in the
`grid_manager_config_version` key which should increment whenever the
`grid_manager_config_version` key which will increment whenever the
document schema changes.
@ -220,7 +224,8 @@ Enrolling a Client: Config
--------------------------
You may instruct a Tahoe client to use only storage servers from given
Grid Managers. If there are no such keys, any servers are used. If
Grid Managers. If there are no such keys, any servers are used
(but see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3979). If
there are one or more keys, the client will only upload to a storage
server that has a valid certificate (from any of the keys).
@ -333,5 +338,5 @@ uploads should now fail (so ``tahoe put`` will fail) because they
won't use storage2 and thus can't "achieve happiness".
A proposal to expose more information about Grid Manager and
certifcate status in the Welcome page is discussed in
certificate status in the Welcome page is discussed in
https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3506

120
docs/proposed/http-storage-node-protocol.rst
View File

@ -30,12 +30,12 @@ Glossary
introducer
a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers
fURL
:ref:`fURLs <fURLs>`
a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol
(the storage service is an example of such an object)
NURL
a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap
:ref:`NURLs <NURLs>`
a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap
swissnum
a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service
@ -395,8 +395,8 @@ Encoding
General
~~~~~~~
``GET /v1/version``
!!!!!!!!!!!!!!!!!!!
``GET /storage/v1/version``
!!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve information about the version of the storage server.
Information is returned as an encoded mapping.
@ -409,14 +409,13 @@ For example::
"tolerates-immutable-read-overrun": true,
"delete-mutable-shares-with-zero-length-writev": true,
"fills-holes-with-zero-bytes": true,
"prevents-read-past-end-of-share-data": true,
"gbs-anonymous-storage-url": "pb://...#v=1"
"prevents-read-past-end-of-share-data": true
},
"application-version": "1.13.0"
}
``PUT /v1/lease/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``PUT /storage/v1/lease/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Either renew or create a new lease on the bucket addressed by ``storage_index``.
@ -468,8 +467,8 @@ Immutable
Writing
~~~~~~~
``POST /v1/immutable/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``POST /storage/v1/immutable/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Initialize an immutable storage index with some buckets.
The buckets may have share data written to them once.
@ -504,7 +503,7 @@ Handling repeat calls:
Discussion
``````````
We considered making this ``POST /v1/immutable`` instead.
We considered making this ``POST /storage/v1/immutable`` instead.
The motivation was to keep *storage index* out of the request URL.
Request URLs have an elevated chance of being logged by something.
We were concerned that having the *storage index* logged may increase some risks.
@ -539,8 +538,8 @@ Rejected designs for upload secrets:
it must contain randomness.
Randomness means there is no need to have a secret per share, since adding share-specific content to randomness doesn't actually make the secret any better.
``PATCH /v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``PATCH /storage/v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Write data for the indicated share.
The share number must belong to the storage index.
@ -580,24 +579,6 @@ Responses:
the response is ``CONFLICT``.
At this point the only thing to do is abort the upload and start from scratch (see below).
``PUT /v1/immutable/:storage_index/:share_number/abort``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
This cancels an *in-progress* upload.
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response code:
* When the upload is still in progress and therefore the abort has succeeded,
the response is ``OK``.
Future uploads can start from scratch with no pre-existing upload state stored on the server.
* If the uploaded has already finished, the response is 405 (Method Not Allowed)
and no change is made.
Discussion
``````````
@ -616,8 +597,27 @@ From RFC 7231::
PATCH method defined in [RFC5789]).
``POST /v1/immutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
This cancels an *in-progress* upload.
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response code:
* When the upload is still in progress and therefore the abort has succeeded,
the response is ``OK``.
Future uploads can start from scratch with no pre-existing upload state stored on the server.
* If the uploaded has already finished, the response is 405 (Method Not Allowed)
and no change is made.
``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Advise the server the data read from the indicated share was corrupt. The
request body includes an human-meaningful text string with details about the
@ -625,7 +625,7 @@ corruption. It also includes potentially important details about the share.
For example::
{"reason": u"expected hash abcd, got hash efgh"}
{"reason": "expected hash abcd, got hash efgh"}
.. share-type, storage-index, and share-number are inferred from the URL
@ -635,8 +635,8 @@ couldn't be found.
Reading
~~~~~~~
``GET /v1/immutable/:storage_index/shares``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``GET /storage/v1/immutable/:storage_index/shares``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve a list (semantically, a set) indicating all shares available for the
indicated storage index. For example::
@ -645,8 +645,8 @@ indicated storage index. For example::
An unknown storage index results in an empty list.
``GET /v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``GET /storage/v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Read a contiguous sequence of bytes from one share in one bucket.
The response body is the raw share data (i.e., ``application/octet-stream``).
@ -686,8 +686,8 @@ Mutable
Writing
~~~~~~~
``POST /v1/mutable/:storage_index/read-test-write``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``POST /storage/v1/mutable/:storage_index/read-test-write``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
General purpose read-test-and-write operation for mutable storage indexes.
A mutable storage index is also called a "slot"
@ -742,18 +742,18 @@ As a result, if there is no data at all, an empty bytestring is returned no matt
Reading
~~~~~~~
``GET /v1/mutable/:storage_index/shares``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``GET /storage/v1/mutable/:storage_index/shares``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve a set indicating all shares available for the indicated storage index.
For example (this is shown as list, since it will be list for JSON, but will be set for CBOR)::
[1, 5]
``GET /v1/mutable/:storage_index/:share_number``
``GET /storage/v1/mutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Read data from the indicated mutable shares, just like ``GET /v1/immutable/:storage_index``
Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``
The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
Interpretation and response behavior is as specified in RFC 7233 § 4.1.
@ -765,8 +765,8 @@ The resulting ``Content-Range`` header will be consistent with the returned data
If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
``POST /v1/mutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
``POST /storage/v1/mutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Advise the server the data read from the indicated share was corrupt.
Just like the immutable version.
@ -779,7 +779,7 @@ Immutable Data
1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded::
POST /v1/immutable/AAAAAAAAAAAAAAAA
POST /storage/v1/immutable/AAAAAAAAAAAAAAAA
Authorization: Tahoe-LAFS nurl-swissnum
X-Tahoe-Authorization: lease-renew-secret efgh
X-Tahoe-Authorization: lease-cancel-secret jjkl
@ -792,23 +792,25 @@ Immutable Data
#. Upload the content for immutable share ``7``::
PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
Authorization: Tahoe-LAFS nurl-swissnum
Content-Range: bytes 0-15/48
X-Tahoe-Authorization: upload-secret xyzf
<first 16 bytes of share data>
200 OK
{ "required": [ {"begin": 16, "end": 48 } ] }
PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
Authorization: Tahoe-LAFS nurl-swissnum
Content-Range: bytes 16-31/48
X-Tahoe-Authorization: upload-secret xyzf
<second 16 bytes of share data>
200 OK
{ "required": [ {"begin": 32, "end": 48 } ] }
PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7
PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
Authorization: Tahoe-LAFS nurl-swissnum
Content-Range: bytes 32-47/48
X-Tahoe-Authorization: upload-secret xyzf
@ -818,16 +820,17 @@ Immutable Data
#. Download the content of the previously uploaded immutable share ``7``::
GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7
GET /storage/v1/immutable/AAAAAAAAAAAAAAAA?share=7
Authorization: Tahoe-LAFS nurl-swissnum
Range: bytes=0-47
200 OK
Content-Range: bytes 0-47/48
<complete 48 bytes of previously uploaded data>
#. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``::
PUT /v1/lease/AAAAAAAAAAAAAAAA
PUT /storage/v1/lease/AAAAAAAAAAAAAAAA
Authorization: Tahoe-LAFS nurl-swissnum
X-Tahoe-Authorization: lease-cancel-secret jjkl
X-Tahoe-Authorization: lease-renew-secret efgh
@ -842,7 +845,7 @@ The special test vector of size 1 but empty bytes will only pass
if there is no existing share,
otherwise it will read a byte which won't match `b""`::
POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
Authorization: Tahoe-LAFS nurl-swissnum
X-Tahoe-Authorization: write-enabler abcd
X-Tahoe-Authorization: lease-cancel-secret efgh
@ -874,7 +877,7 @@ otherwise it will read a byte which won't match `b""`::
#. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail)::
POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
POST /storage/v1/mutable/BBBBBBBBBBBBBBBB/read-test-write
Authorization: Tahoe-LAFS nurl-swissnum
X-Tahoe-Authorization: write-enabler abcd
X-Tahoe-Authorization: lease-cancel-secret efgh
@ -906,14 +909,17 @@ otherwise it will read a byte which won't match `b""`::
#. Download the contents of share number ``3``::
GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10
GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3
Authorization: Tahoe-LAFS nurl-swissnum
Range: bytes=0-16
200 OK
Content-Range: bytes 0-15/16
<complete 16 bytes of previously uploaded data>
#. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``::
PUT /v1/lease/BBBBBBBBBBBBBBBB
PUT /storage/v1/lease/BBBBBBBBBBBBBBBB
Authorization: Tahoe-LAFS nurl-swissnum
X-Tahoe-Authorization: lease-cancel-secret efgh
X-Tahoe-Authorization: lease-renew-secret ijkl

29
docs/running.rst
View File

@ -124,6 +124,35 @@ Tahoe-LAFS.
.. _magic wormhole: https://magic-wormhole.io/
Multiple Instances
------------------
Running multiple instances against the same configuration directory isn't supported.
This will lead to undefined behavior and could corrupt the configuration or state.
We attempt to avoid this situation with a "pidfile"-style file in the config directory called ``running.process``.
There may be a parallel file called ``running.process.lock`` in existence.
The ``.lock`` file exists to make sure only one process modifies ``running.process`` at once.
The lock file is managed by the `lockfile <https://pypi.org/project/lockfile/>`_ library.
If you wish to make use of ``running.process`` for any reason you should also lock it and follow the semantics of lockfile.
If ``running.process`` exists then it contains the PID and the creation-time of the process.
When no such file exists, there is no other process running on this configuration.
If there is a ``running.process`` file, it may be a leftover file or it may indicate that another process is running against this config.
To tell the difference, determine if the PID in the file exists currently.
If it does, check the creation-time of the process versus the one in the file.
If these match, there is another process currently running and using this config.
Otherwise, the file is stale -- it should be removed before starting Tahoe-LAFS.
Some example Python code to check the above situations:
.. literalinclude:: check_running.py
A note about small grids
------------------------

42
docs/specifications/url.rst
View File

@ -7,6 +7,8 @@ These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to r
An attempt is also made to outline the rationale for certain choices about these URLs.
The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs.
.. _furls:
Background
----------
@ -31,6 +33,8 @@ The client's use of the swissnum is what allows the server to authorize the clie
.. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number
.. _NURLs:
NURLs
-----
@ -47,27 +51,27 @@ This can be considered to expand to "**N**\ ew URLs" or "Authe\ **N**\ ticating
The anticipated use for a **NURL** will still be to establish a TLS connection to a peer.
The protocol run over that TLS connection could be Foolscap though it is more likely to be an HTTP-based protocol (such as GBS).
Unlike fURLs, only a single net-loc is included, for consistency with other forms of URLs.
As a result, multiple NURLs may be available for a single server.
Syntax
------
The EBNF for a NURL is as follows::
nurl = scheme, hash, "@", net-loc-list, "/", swiss-number, [ version1 ]
scheme = "pb://"
nurl = tcp-nurl | tor-nurl | i2p-nurl
tcp-nurl = "pb://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ]
tor-nurl = "pb+tor://", hash, "@", tcp-loc, "/", swiss-number, [ version1 ]
i2p-nurl = "pb+i2p://", hash, "@", i2p-loc, "/", swiss-number, [ version1 ]
hash = unreserved
net-loc-list = net-loc, [ { ",", net-loc } ]
net-loc = tcp-loc | tor-loc | i2p-loc
tcp-loc = [ "tcp:" ], hostname, [ ":" port ]
tor-loc = "tor:", hostname, [ ":" port ]
i2p-loc = "i2p:", i2p-addr, [ ":" port ]
i2p-addr = { unreserved }, ".i2p"
tcp-loc = hostname, [ ":" port ]
hostname = domain | IPv4address | IPv6address
i2p-loc = i2p-addr, [ ":" port ]
i2p-addr = { unreserved }, ".i2p"
swiss-number = segment
version1 = "#v=1"
@ -87,11 +91,13 @@ These differences are separated into distinct versions.
Version 0
---------
A Foolscap fURL is considered the canonical definition of a version 0 NURL.
In theory, a Foolscap fURL with a single netloc is considered the canonical definition of a version 0 NURL.
Notably,
the hash component is defined as the base32-encoded SHA1 hash of the DER form of an x509v3 certificate.
A version 0 NURL is identified by the absence of the ``v=1`` fragment.
In practice, real world fURLs may have more than one netloc, so lack of version fragment will likely just involve dispatching the fURL to a different parser.
Examples
~~~~~~~~
@ -103,11 +109,8 @@ Version 1
The hash component of a version 1 NURL differs in three ways from the prior version.
1. The hash function used is SHA3-224 instead of SHA1.
The security of SHA1 `continues to be eroded`_.
Contrariwise SHA3 is currently the most recent addition to the SHA family by NIST.
The 224 bit instance is chosen to keep the output short and because it offers greater collision resistance than SHA1 was thought to offer even at its inception
(prior to security research showing actual collision resistance is lower).
1. The hash function used is SHA-256, to match RFC 7469.
The security of SHA1 `continues to be eroded`_; Latacora `SHA-2`_.
2. The hash is computed over the certificate's SPKI instead of the whole certificate.
This allows certificate re-generation so long as the public key remains the same.
This is useful to allow contact information to be updated or extension of validity period.
@ -122,7 +125,7 @@ The hash component of a version 1 NURL differs in three ways from the prior vers
*all* certificate fields should be considered within the context of the relationship identified by the SPKI hash.
3. The hash is encoded using urlsafe-base64 (without padding) instead of base32.
This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 224 bit hash.
This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 256 bit hash.
A version 1 NURL is identified by the presence of the ``v=1`` fragment.
Though the length of the hash string (38 bytes) could also be used to differentiate it from a version 0 NURL,
@ -140,7 +143,8 @@ Examples
* ``pb://azEu8vlRpnEeYm0DySQDeNY3Z2iJXHC_bsbaAw@localhost:47877/64i4aokv4ej#v=1``
.. _`continues to be eroded`: https://en.wikipedia.org/wiki/SHA-1#Cryptanalysis_and_validation
.. _`explored by the web community`: https://www.imperialviolet.org/2011/05/04/pinning.html
.. _`SHA-2`: https://latacora.micro.blog/2018/04/03/cryptographic-right-answers.html
.. _`explored by the web community`: https://www.rfc-editor.org/rfc/rfc7469
.. _Foolscap: https://github.com/warner/foolscap
.. [1] ``foolscap.furl.decode_furl`` is taken as the canonical definition of the syntax of a fURL.

38
integration/conftest.py
View File

@ -1,15 +1,6 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
import shutil
from time import sleep
@ -42,7 +33,6 @@ from .util import (
_tahoe_runner_optional_coverage,
await_client_ready,
cli,
_run_node,
generate_ssh_key,
block_with_timeout,
)
@ -64,6 +54,29 @@ def pytest_addoption(parser):
"--coverage", action="store_true", dest="coverage",
help="Collect coverage statistics",
)
parser.addoption(
"--force-foolscap", action="store_true", default=False,
dest="force_foolscap",
help=("If set, force Foolscap only for the storage protocol. " +
"Otherwise HTTP will be used.")
)
parser.addoption(
"--runslow", action="store_true", default=False,
dest="runslow",
help="If set, run tests marked as slow.",
)
def pytest_collection_modifyitems(session, config, items):
if not config.option.runslow:
# The --runslow option was not given; keep only collected items not
# marked as slow.
items[:] = [
item
for item
in items
if item.get_closest_marker("slow") is None
]
@pytest.fixture(autouse=True, scope='session')
def eliot_logging():
@ -300,10 +313,9 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
pytest_twisted.blockon(process.restart_async(reactor, request))
await_client_ready(process)
print(f"Alice pid: {process.transport.pid}")
return process

73
integration/test_get_put.py
View File

@ -3,11 +3,14 @@ Integration tests for getting and putting files, including reading from stdin
and stdout.
"""
from subprocess import Popen, PIPE
from subprocess import Popen, PIPE, check_output, check_call
import sys
import pytest
from pytest_twisted import ensureDeferred
from twisted.internet import reactor
from .util import run_in_thread, cli
from .util import run_in_thread, cli, reconfigure
DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11"
try:
@ -62,3 +65,69 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir):
)
assert p.stdout.read() == DATA
assert p.wait() == 0
def test_large_file(alice, get_put_alias, tmp_path):
"""
It's possible to upload and download a larger file.
We avoid stdin/stdout since that's flaky on Windows.
"""
tempfile = tmp_path / "file"
with tempfile.open("wb") as f:
f.write(DATA * 1_000_000)
cli(alice, "put", str(tempfile), "getput:largefile")
outfile = tmp_path / "out"
check_call(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:largefile", str(outfile)],
)
assert outfile.read_bytes() == tempfile.read_bytes()
@pytest.mark.skipif(
sys.platform.startswith("win"),
reason="reconfigure() has issues on Windows"
)
@ensureDeferred
async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
"""
Tahoe-LAFS used to have a default max segment size of 128KB, and is now
1MB. Test that an upload created when 128KB was the default can be
downloaded with 1MB as the default (i.e. old uploader, new downloader), and
vice versa, (new uploader, old downloader).
"""
tempfile = tmpdir.join("file")
large_data = DATA * 100_000
assert len(large_data) > 2 * 1024 * 1024
with tempfile.open("wb") as f:
f.write(large_data)
async def set_segment_size(segment_size):
await reconfigure(
reactor,
request,
alice,
(1, 1, 1),
None,
max_segment_size=segment_size
)
# 1. Upload file 1 with default segment size set to 1MB
await set_segment_size(1024 * 1024)
cli(alice, "put", str(tempfile), "getput:seg1024kb")
# 2. Download file 1 with default segment size set to 128KB
await set_segment_size(128 * 1024)
assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
)
# 3. Upload file 2 with default segment size set to 128KB
cli(alice, "put", str(tempfile), "getput:seg128kb")
# 4. Download file 2 with default segment size set to 1MB
await set_segment_size(1024 * 1024)
assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
)

7
integration/test_i2p.py
View File

@ -55,15 +55,18 @@ def i2p_network(reactor, temp_dir, request):
proto,
which("docker"),
(
"docker", "run", "-p", "7656:7656", "purplei2p/i2pd",
"docker", "run", "-p", "7656:7656", "purplei2p/i2pd:release-2.45.1",
# Bad URL for reseeds, so it can't talk to other routers.
"--reseed.urls", "http://localhost:1/",
# Make sure we see the "ephemeral keys message"
"--log=stdout",
"--loglevel=info"
),
)
def cleanup():
try:
proto.transport.signalProcess("KILL")
proto.transport.signalProcess("INT")
util.block_with_timeout(proto.exited, reactor)
except ProcessExitedAlready:
pass

121
integration/test_vectors.py Normal file
View File

@ -0,0 +1,121 @@
"""
Verify certain results against test vectors with well-known results.
"""
from __future__ import annotations
from functools import partial
from typing import AsyncGenerator, Iterator
from itertools import starmap, product
from attrs import evolve
from pytest import mark
from pytest_twisted import ensureDeferred
from . import vectors
from .vectors import parameters
from .util import reconfigure, upload, TahoeProcess
@mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS)
def test_convergence(convergence):
"""
Convergence secrets are 16 bytes.
"""
assert isinstance(convergence, bytes), "Convergence secret must be bytes"
assert len(convergence) == 16, "Convergence secret must by 16 bytes"
@mark.slow
@mark.parametrize('case,expected', vectors.capabilities.items())
@ensureDeferred
async def test_capability(reactor, request, alice, case, expected):
"""
The capability that results from uploading certain well-known data
with certain well-known parameters results in exactly the previously
computed value.
"""
# rewrite alice's config to match params and convergence
await reconfigure(
reactor, request, alice, (1, case.params.required, case.params.total), case.convergence, case.segment_size)
# upload data in the correct format
actual = upload(alice, case.fmt, case.data)
# compare the resulting cap to the expected result
assert actual == expected
@ensureDeferred
async def skiptest_generate(reactor, request, alice):
"""
This is a helper for generating the test vectors.
You can re-generate the test vectors by fixing the name of the test and
running it. Normally this test doesn't run because it ran once and we
captured its output. Other tests run against that output and we want them
to run against the results produced originally, not a possibly
ever-changing set of outputs.
"""
space = starmap(
# segment_size could be a parameter someday but it's not easy to vary
# using the Python implementation so it isn't one for now.
partial(vectors.Case, segment_size=parameters.SEGMENT_SIZE),
product(
parameters.ZFEC_PARAMS,
parameters.CONVERGENCE_SECRETS,
parameters.OBJECT_DESCRIPTIONS,
parameters.FORMATS,
),
)
iterresults = generate(reactor, request, alice, space)
results = []
async for result in iterresults:
# Accumulate the new result
results.append(result)
# Then rewrite the whole output file with the new accumulator value.
# This means that if we fail partway through, we will still have
# recorded partial results -- instead of losing them all.
vectors.save_capabilities(results)
async def generate(
reactor,
request,
alice: TahoeProcess,
cases: Iterator[vectors.Case],
) -> AsyncGenerator[[vectors.Case, str], None]:
"""
Generate all of the test vectors using the given node.
:param reactor: The reactor to use to restart the Tahoe-LAFS node when it
needs to be reconfigured.
:param request: The pytest request object to use to arrange process
cleanup.
:param format: The name of the encryption/data format to use.
:param alice: The Tahoe-LAFS node to use to generate the test vectors.
:param case: The inputs for which to generate a value.
:return: The capability for the case.
"""
# Share placement doesn't affect the resulting capability. For maximum
# reliability of this generator, be happy if we can put shares anywhere
happy = 1
for case in cases:
await reconfigure(
reactor,
request,
alice,
(happy, case.params.required, case.params.total),
case.convergence,
case.segment_size
)
# Give the format a chance to make an RSA key if it needs it.
case = evolve(case, fmt=case.fmt.customize())
cap = upload(alice, case.fmt, case.data)
yield case, cap

24
integration/test_web.py
View File

@ -7,18 +7,9 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did
at the time of testing' -- not necessarily a cohesive idea of what the
WebAPI *should* do in every situation. It's not clear the latter
exists anywhere, however.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import time
from urllib.parse import unquote as url_unquote, quote as url_quote
@ -32,6 +23,7 @@ import requests
import html5lib
from bs4 import BeautifulSoup
from pytest_twisted import ensureDeferred
def test_index(alice):
"""
@ -252,10 +244,18 @@ def test_status(alice):
assert found_download, "Failed to find the file we downloaded in the status-page"
def test_directory_deep_check(alice):
@ensureDeferred
async def test_directory_deep_check(reactor, request, alice):
"""
use deep-check and confirm the result pages work
"""
# Make sure the node is configured compatibly with expectations of this
# test.
happy = 3
required = 2
total = 4
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
# create a directory
resp = requests.post(
@ -313,7 +313,7 @@ def test_directory_deep_check(alice):
)
def check_repair_data(checkdata):
assert checkdata["healthy"] is True
assert checkdata["healthy"]
assert checkdata["count-happiness"] == 4
assert checkdata["count-good-share-hosts"] == 4
assert checkdata["count-shares-good"] == 4

308
integration/util.py
View File

@ -1,22 +1,19 @@
"""
Ported to Python 3.
General functionality useful for the implementation of integration tests.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from contextlib import contextmanager
from typing import Any
from typing_extensions import Literal
from tempfile import NamedTemporaryFile
import sys
import time
import json
from os import mkdir, environ
from os.path import exists, join
from io import StringIO, BytesIO
from functools import partial
from subprocess import check_output
from twisted.python.filepath import (
@ -26,18 +23,30 @@ from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
from twisted.internet.threads import deferToThread
from twisted.internet.interfaces import IProcessTransport, IReactorProcess
from attrs import frozen, evolve
import requests
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import (
Encoding,
PrivateFormat,
NoEncryption,
)
from paramiko.rsakey import RSAKey
from boltons.funcutils import wraps
from allmydata.util import base32
from allmydata.util.configutil import (
get_config,
set_config,
write_config,
)
from allmydata import client
from allmydata.interfaces import DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE
import pytest_twisted
@ -166,9 +175,40 @@ class _MagicTextProtocol(ProcessProtocol):
sys.stdout.write(data)
def _cleanup_tahoe_process(tahoe_transport, exited):
def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
"""
Terminate the given process with a kill signal (SIGKILL on POSIX,
If the given process transport seems to still be associated with a
running process, send a SIGTERM to that process.
:param transport: The transport to use.
:param allow_missing: If ``True`` then it is not an error for the
transport to have no associated process. Otherwise, an exception will
be raised in that case.
:raise: ``ValueError`` if ``allow_missing`` is ``False`` and the transport
has no process.
"""
if transport.pid is None:
if allow_missing:
print("Process already cleaned up and that's okay.")
return
else:
raise ValueError("Process is not running")
print("signaling {} with TERM".format(transport.pid))
try:
transport.signalProcess('TERM')
except ProcessExitedAlready:
# The transport object thought it still had a process but the real OS
# process has already exited. That's fine. We accomplished what we
# wanted to. We don't care about ``allow_missing`` here because
# there's no way we could have known the real OS process already
# exited.
pass
def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False):
"""
Terminate the given process with a kill signal (SIGTERM on POSIX,
TerminateProcess on Windows).
:param tahoe_transport: The `IProcessTransport` representing the process.
@ -177,14 +217,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited):
:return: After the process has exited.
"""
from twisted.internet import reactor
try:
print("signaling {} with TERM".format(tahoe_transport.pid))
tahoe_transport.signalProcess('TERM')
print("signaled, blocking on exit {}".format(exited))
block_with_timeout(exited, reactor)
print("exited, goodbye")
except ProcessExitedAlready:
pass
_cleanup_process_async(tahoe_transport, allow_missing=allow_missing)
print(f"signaled, blocking on exit {exited}")
block_with_timeout(exited, reactor)
print("exited, goodbye")
def run_tahoe(reactor, request, *args, **kwargs):
@ -245,8 +281,33 @@ class TahoeProcess(object):
def kill(self):
"""Kill the process, block until it's done."""
print(f"TahoeProcess.kill({self.transport.pid} / {self.node_dir})")
_cleanup_tahoe_process(self.transport, self.transport.exited)
def kill_async(self):
"""
Kill the process, return a Deferred that fires when it's done.
"""
print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})")
_cleanup_process_async(self.transport, allow_missing=False)
return self.transport.exited
def restart_async(self, reactor: IReactorProcess, request: Any) -> Deferred:
"""
Stop and then re-start the associated process.
:return: A Deferred that fires after the new process is ready to
handle requests.
"""
d = self.kill_async()
d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False))
def got_new_process(proc):
# Grab the new transport since the one we had before is no longer
# valid after the stop/start cycle.
self._process_transport = proc.transport
d.addCallback(got_new_process)
return d
def __str__(self):
return "<TahoeProcess in '{}'>".format(self._node_dir)
@ -275,19 +336,17 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
)
transport.exited = protocol.exited
tahoe_process = TahoeProcess(
transport,
node_dir,
)
if finalize:
request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
request.addfinalizer(tahoe_process.kill)
# XXX abusing the Deferred; should use .when_magic_seen() pattern
def got_proto(proto):
transport._protocol = proto
return TahoeProcess(
transport,
node_dir,
)
protocol.magic_seen.addCallback(got_proto)
return protocol.magic_seen
d = protocol.magic_seen
d.addCallback(lambda ignored: tahoe_process)
return d
def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
@ -338,6 +397,20 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
u'log_gatherer.furl',
flog_gatherer.furl,
)
force_foolscap = request.config.getoption("force_foolscap")
assert force_foolscap in (True, False)
set_config(
config,
'storage',
'force_foolscap',
str(force_foolscap),
)
set_config(
config,
'client',
'force_foolscap',
str(force_foolscap),
)
write_config(FilePath(config_path), config)
created_d.addCallback(created)
@ -610,3 +683,178 @@ def run_in_thread(f):
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test
@frozen
class CHK:
"""
Represent the CHK encoding sufficiently to run a ``tahoe put`` command
using it.
"""
kind = "chk"
max_shares = 256
def customize(self) -> CHK:
# Nothing to do.
return self
@classmethod
def load(cls, params: None) -> CHK:
assert params is None
return cls()
def to_json(self) -> None:
return None
@contextmanager
def to_argv(self) -> None:
yield []
@frozen
class SSK:
"""
Represent the SSK encodings (SDMF and MDMF) sufficiently to run a
``tahoe put`` command using one of them.
"""
kind = "ssk"
# SDMF and MDMF encode share counts (N and k) into the share itself as an
# unsigned byte. They could have encoded (share count - 1) to fit the
# full range supported by ZFEC into the unsigned byte - but they don't.
# So 256 is inaccessible to those formats and we set the upper bound at
# 255.
max_shares = 255
name: Literal["sdmf", "mdmf"]
key: None | bytes
@classmethod
def load(cls, params: dict) -> SSK:
assert params.keys() == {"format", "mutable", "key"}
return cls(params["format"], params["key"].encode("ascii"))
def customize(self) -> SSK:
"""
Return an SSK with a newly generated random RSA key.
"""
return evolve(self, key=generate_rsa_key())
def to_json(self) -> dict[str, str]:
return {
"format": self.name,
"mutable": None,
"key": self.key.decode("ascii"),
}
@contextmanager
def to_argv(self) -> None:
with NamedTemporaryFile() as f:
f.write(self.key)
f.flush()
yield [f"--format={self.name}", "--mutable", f"--private-key-path={f.name}"]
def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str:
"""
Upload the given data to the given node.
:param alice: The node to upload to.
:param fmt: The name of the format for the upload. CHK, SDMF, or MDMF.
:param data: The data to upload.
:return: The capability for the uploaded data.
"""
with NamedTemporaryFile() as f:
f.write(data)
f.flush()
with fmt.to_argv() as fmt_argv:
argv = [alice, "put"] + fmt_argv + [f.name]
return cli(*argv).decode("utf-8").strip()
async def reconfigure(reactor, request, node: TahoeProcess,
params: tuple[int, int, int],
convergence: None | bytes,
max_segment_size: None | int = None) -> None:
"""
Reconfigure a Tahoe-LAFS node with different ZFEC parameters and
convergence secret.
TODO This appears to have issues on Windows.
If the current configuration is different from the specified
configuration, the node will be restarted so it takes effect.
:param reactor: A reactor to use to restart the process.
:param request: The pytest request object to use to arrange process
cleanup.
:param node: The Tahoe-LAFS node to reconfigure.
:param params: The ``happy``, ``needed``, and ``total`` ZFEC encoding
parameters.
:param convergence: If given, the convergence secret. If not given, the
existing convergence secret will be left alone.
:return: ``None`` after the node configuration has been rewritten, the
node has been restarted, and the node is ready to provide service.
"""
happy, needed, total = params
config = node.get_config()
changed = False
cur_happy = int(config.get_config("client", "shares.happy"))
cur_needed = int(config.get_config("client", "shares.needed"))
cur_total = int(config.get_config("client", "shares.total"))
if (happy, needed, total) != (cur_happy, cur_needed, cur_total):
changed = True
config.set_config("client", "shares.happy", str(happy))
config.set_config("client", "shares.needed", str(needed))
config.set_config("client", "shares.total", str(total))
if convergence is not None:
cur_convergence = config.get_private_config("convergence").encode("ascii")
if base32.a2b(cur_convergence) != convergence:
changed = True
config.write_private_config("convergence", base32.b2a(convergence))
if max_segment_size is not None:
cur_segment_size = int(config.get_config("client", "shares._max_immutable_segment_size_for_testing", DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE))
if cur_segment_size != max_segment_size:
changed = True
config.set_config(
"client",
"shares._max_immutable_segment_size_for_testing",
str(max_segment_size)
)
if changed:
# TODO reconfigure() seems to have issues on Windows. If you need to
# use it there, delete this assert and try to figure out what's going
# on...
assert not sys.platform.startswith("win")
# restart the node
print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
await node.restart_async(reactor, request)
print("Restarted. Waiting for ready state.")
await_client_ready(node)
print("Ready.")
else:
print("Config unchanged, not restarting.")
def generate_rsa_key() -> bytes:
"""
Generate a 2048 bit RSA key suitable for use with SSKs.
"""
return rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
).private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
)

30
integration/vectors/__init__.py Normal file
View File

@ -0,0 +1,30 @@
__all__ = [
"DATA_PATH",
"CURRENT_VERSION",
"MAX_SHARES",
"Case",
"Sample",
"SeedParam",
"encode_bytes",
"save_capabilities",
"capabilities",
]
from .vectors import (
DATA_PATH,
CURRENT_VERSION,
Case,
Sample,
SeedParam,
encode_bytes,
save_capabilities,
capabilities,
)
from .parameters import (
MAX_SHARES,
)

58
integration/vectors/model.py Normal file
View File

@ -0,0 +1,58 @@
"""
Simple data type definitions useful in the definition/verification of test
vectors.
"""
from __future__ import annotations
from attrs import frozen
# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares!
# Represent max symbolically and resolve it when we know what format we're
# dealing with.
MAX_SHARES = "max"
@frozen
class Sample:
"""
Some instructions for building a long byte string.
:ivar seed: Some bytes to repeat some times to produce the string.
:ivar length: The length of the desired byte string.
"""
seed: bytes
length: int
@frozen
class Param:
"""
Some ZFEC parameters.
"""
required: int
total: int
@frozen
class SeedParam:
"""
Some ZFEC parameters, almost.
:ivar required: The number of required shares.
:ivar total: Either the number of total shares or the constant
``MAX_SHARES`` to indicate that the total number of shares should be
the maximum number supported by the object format.
"""
required: int
total: int | str
def realize(self, max_total: int) -> Param:
"""
Create a ``Param`` from this object's values, possibly
substituting the given real value for total if necessary.
:param max_total: The value to use to replace ``MAX_SHARES`` if
necessary.
"""
if self.total == MAX_SHARES:
return Param(self.required, max_total)
return Param(self.required, self.total)

93
integration/vectors/parameters.py Normal file
View File

@ -0,0 +1,93 @@
"""
Define input parameters for test vector generation.
:ivar CONVERGENCE_SECRETS: Convergence secrets.
:ivar SEGMENT_SIZE: The single segment size that the Python implementation
currently supports without a lot of refactoring.
:ivar OBJECT_DESCRIPTIONS: Small objects with instructions which can be
expanded into a possibly large byte string. These are intended to be used
as plaintext inputs.
:ivar ZFEC_PARAMS: Input parameters to ZFEC.
:ivar FORMATS: Encoding/encryption formats.
"""
from __future__ import annotations
from hashlib import sha256
from .model import MAX_SHARES
from .vectors import Sample, SeedParam
from ..util import CHK, SSK
def digest(bs: bytes) -> bytes:
"""
Digest bytes to bytes.
"""
return sha256(bs).digest()
def hexdigest(bs: bytes) -> str:
"""
Digest bytes to text.
"""
return sha256(bs).hexdigest()
# Just a couple convergence secrets. The only thing we do with this value is
# feed it into a tagged hash. It certainly makes a difference to the output
# but the hash should destroy any structure in the input so it doesn't seem
# like there's a reason to test a lot of different values.
CONVERGENCE_SECRETS: list[bytes] = [
b"aaaaaaaaaaaaaaaa",
digest(b"Hello world")[:16],
]
SEGMENT_SIZE: int = 128 * 1024
# Exercise at least a handful of different sizes, trying to cover:
#
# 1. Some cases smaller than one "segment" (128k).
# This covers shrinking of some parameters to match data size.
# This includes one case of the smallest possible CHK.
#
# 2. Some cases right on the edges of integer segment multiples.
# Because boundaries are tricky.
#
# 4. Some cases that involve quite a few segments.
# This exercises merkle tree construction more thoroughly.
#
# See ``stretch`` for construction of the actual test data.
OBJECT_DESCRIPTIONS: list[Sample] = [
# The smallest possible. 55 bytes and smaller are LIT.
Sample(b"a", 56),
Sample(b"a", 1024),
Sample(b"c", 4096),
Sample(digest(b"foo"), SEGMENT_SIZE - 1),
Sample(digest(b"bar"), SEGMENT_SIZE + 1),
Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1),
Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1),
Sample(digest(b"bazquux"), SEGMENT_SIZE * 32),
Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1),
Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1),
]
ZFEC_PARAMS: list[SeedParam] = [
SeedParam(1, 1),
SeedParam(1, 3),
SeedParam(2, 3),
SeedParam(3, 10),
SeedParam(71, 255),
SeedParam(101, MAX_SHARES),
]
FORMATS: list[CHK | SSK] = [
CHK(),
# These start out unaware of a key but various keys will be supplied
# during generation.
SSK(name="sdmf", key=None),
SSK(name="mdmf", key=None),
]

18002
integration/vectors/test_vectors.yaml Executable file
View File

File diff suppressed because it is too large Load Diff

155
integration/vectors/vectors.py Normal file
View File

@ -0,0 +1,155 @@
"""
A module that loads pre-generated test vectors.
:ivar DATA_PATH: The path of the file containing test vectors.
:ivar capabilities: The capability test vectors.
"""
from __future__ import annotations
from typing import TextIO
from attrs import frozen
from yaml import safe_load, safe_dump
from base64 import b64encode, b64decode
from twisted.python.filepath import FilePath
from .model import Param, Sample, SeedParam
from ..util import CHK, SSK
DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml")
# The version of the persisted test vector data this code can interpret.
CURRENT_VERSION: str = "2023-01-16.2"
@frozen
class Case:
"""
Represent one case for which we want/have a test vector.
"""
seed_params: Param
convergence: bytes
seed_data: Sample
fmt: CHK | SSK
segment_size: int
@property
def data(self):
return stretch(self.seed_data.seed, self.seed_data.length)
@property
def params(self):
return self.seed_params.realize(self.fmt.max_shares)
def encode_bytes(b: bytes) -> str:
"""
Base64 encode some bytes to text so they are representable in JSON.
"""
return b64encode(b).decode("ascii")
def decode_bytes(b: str) -> bytes:
"""
Base64 decode some text to bytes.
"""
return b64decode(b.encode("ascii"))
def stretch(seed: bytes, size: int) -> bytes:
"""
Given a simple description of a byte string, return the byte string
itself.
"""
assert isinstance(seed, bytes)
assert isinstance(size, int)
assert size > 0
assert len(seed) > 0
multiples = size // len(seed) + 1
return (seed * multiples)[:size]
def save_capabilities(results: list[tuple[Case, str]], path: FilePath = DATA_PATH) -> None:
"""
Save some test vector cases and their expected values.
This is logically the inverse of ``load_capabilities``.
"""
path.setContent(safe_dump({
"version": CURRENT_VERSION,
"vector": [
{
"convergence": encode_bytes(case.convergence),
"format": {
"kind": case.fmt.kind,
"params": case.fmt.to_json(),
},
"sample": {
"seed": encode_bytes(case.seed_data.seed),
"length": case.seed_data.length,
},
"zfec": {
"segmentSize": case.segment_size,
"required": case.params.required,
"total": case.params.total,
},
"expected": cap,
}
for (case, cap)
in results
],
}).encode("ascii"))
def load_format(serialized: dict) -> CHK | SSK:
"""
Load an encrypted object format from a simple description of it.
:param serialized: A ``dict`` describing either CHK or SSK, possibly with
some parameters.
"""
if serialized["kind"] == "chk":
return CHK.load(serialized["params"])
elif serialized["kind"] == "ssk":
return SSK.load(serialized["params"])
else:
raise ValueError(f"Unrecognized format: {serialized}")
def load_capabilities(f: TextIO) -> dict[Case, str]:
"""
Load some test vector cases and their expected results from the given
file.
This is logically the inverse of ``save_capabilities``.
"""
data = safe_load(f)
if data is None:
return {}
if data["version"] != CURRENT_VERSION:
print(
f"Current version is {CURRENT_VERSION}; "
f"cannot load version {data['version']} data."
)
return {}
return {
Case(
seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]),
segment_size=case["zfec"]["segmentSize"],
convergence=decode_bytes(case["convergence"]),
seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]),
fmt=load_format(case["format"]),
): case["expected"]
for case
in data["vector"]
}
try:
with DATA_PATH.open() as f:
capabilities: dict[Case, str] = load_capabilities(f)
except FileNotFoundError:
capabilities = {}

95
misc/build_helpers/update-version.py Normal file
View File

@ -0,0 +1,95 @@
#
# this updates the (tagged) version of the software
#
# Any "options" are hard-coded in here (e.g. the GnuPG key to use)
#
author = "meejah <meejah@meejah.ca>"
import sys
import time
from datetime import datetime
from packaging.version import Version
from dulwich.repo import Repo
from dulwich.porcelain import (
tag_list,
tag_create,
status,
)
from twisted.internet.task import (
react,
)
from twisted.internet.defer import (
ensureDeferred,
)
def existing_tags(git):
versions = sorted(
Version(v.decode("utf8").lstrip("tahoe-lafs-"))
for v in tag_list(git)
if v.startswith(b"tahoe-lafs-")
)
return versions
def create_new_version(git):
versions = existing_tags(git)
biggest = versions[-1]
return Version(
"{}.{}.{}".format(
biggest.major,
biggest.minor + 1,
0,
)
)
async def main(reactor):
git = Repo(".")
st = status(git)
if any(st.staged.values()) or st.unstaged:
print("unclean checkout; aborting")
raise SystemExit(1)
v = create_new_version(git)
if "--no-tag" in sys.argv:
print(v)
return
print("Existing tags: {}".format("\n".join(str(x) for x in existing_tags(git))))
print("New tag will be {}".format(v))
# the "tag time" is seconds from the epoch .. we quantize these to
# the start of the day in question, in UTC.
now = datetime.now()
s = now.utctimetuple()
ts = int(
time.mktime(
time.struct_time((s.tm_year, s.tm_mon, s.tm_mday, 0, 0, 0, 0, s.tm_yday, 0))
)
)
tag_create(
repo=git,
tag="tahoe-lafs-{}".format(str(v)).encode("utf8"),
author=author.encode("utf8"),
message="Release {}".format(v).encode("utf8"),
annotated=True,
objectish=b"HEAD",
sign=author.encode("utf8"),
tag_time=ts,
tag_timezone=0,
)
print("Tag created locally, it is not pushed")
print("To push it run something like:")
print(" git push origin {}".format(v))
if __name__ == "__main__":
react(lambda r: ensureDeferred(main(r)))

4
misc/simulators/storage-overhead.py
View File

@ -5,7 +5,7 @@ from __future__ import print_function
import sys, math
from allmydata import uri, storage
from allmydata.immutable import upload
from allmydata.interfaces import DEFAULT_MAX_SEGMENT_SIZE
from allmydata.interfaces import DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE
from allmydata.util import mathutil
def roundup(size, blocksize=4096):
@ -26,7 +26,7 @@ class BigFakeString(object):
def tell(self):
return self.fp
def calc(filesize, params=(3,7,10), segsize=DEFAULT_MAX_SEGMENT_SIZE):
def calc(filesize, params=(3,7,10), segsize=DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE):
num_shares = params[2]
if filesize <= upload.Uploader.URI_LIT_SIZE_THRESHOLD:
urisize = len(uri.LiteralFileURI("A"*filesize).to_string())

36
misc/windows-enospc/passthrough.py Normal file
View File

@ -0,0 +1,36 @@
"""
Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on
Windows. So, this program passes through data from stdin to stdout, using
Windows APIs instead of Unix-y APIs.
"""
from twisted.internet.stdio import StandardIO
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.interfaces import IHalfCloseableProtocol
from twisted.internet.error import ReactorNotRunning
from zope.interface import implementer
@implementer(IHalfCloseableProtocol)
class Passthrough(Protocol):
def readConnectionLost(self):
self.transport.loseConnection()
def writeConnectionLost(self):
try:
reactor.stop()
except ReactorNotRunning:
pass
def dataReceived(self, data):
self.transport.write(data)
def connectionLost(self, reason):
try:
reactor.stop()
except ReactorNotRunning:
pass
std = StandardIO(Passthrough())
reactor.run()

7
mypy.ini
View File

@ -1,3 +1,10 @@
[mypy]
ignore_missing_imports = True
plugins=mypy_zope:plugin
show_column_numbers = True
pretty = True
show_error_codes = True
warn_unused_configs =True
no_implicit_optional = True
warn_redundant_casts = True
strict_equality = True

2
newsfragments/2916.feature
View File

@ -1 +1 @@
A new 'Grid Manager' specification and implementation
Tahoe-LAFS now includes a new "Grid Manager" specification and implementation adding more options to control which storage servers a client will use for uploads.

1
newsfragments/3697.minor
View File

@ -1 +0,0 @@
Added support for Python 3.10. Added support for PyPy3 (3.7 and 3.8, on Linux only).

0
newsfragments/3327.minor → newsfragments/3783.minor
View File

1
newsfragments/3786.minor
View File

@ -1 +0,0 @@
Added re-structured text documentation for the OpenMetrics format statistics endpoint.

8
newsfragments/3828.feature
View File

@ -1,8 +0,0 @@
The implementation of SDMF and MDMF (mutables) now requires RSA keys to be exactly 2048 bits, aligning them with the specification.
Some code existed to allow tests to shorten this and it's
conceptually possible a modified client produced mutables
with different key-sizes. However, the spec says that they
must be 2048 bits. If you happen to have a capability with
a key-size different from 2048 you may use 1.17.1 or earlier
to read the content.

1
newsfragments/3865.incompat
View File

@ -1 +0,0 @@
Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates.

0
newsfragments/3709.minor → newsfragments/3870.minor
View File

1
newsfragments/3873.incompat
View File

@ -1 +0,0 @@
Python 3.7 or later is now required; Python 2 is no longer supported.

0
newsfragments/3788.minor → newsfragments/3874.minor
View File

1
newsfragments/3879.incompat
View File

@ -1 +0,0 @@
Share corruption reports stored on disk are now always encoded in UTF-8.

1
newsfragments/3902.feature Normal file
View File

@ -0,0 +1 @@
The new HTTPS-based storage server is now enabled transparently on the same port as the Foolscap server. This will not have any user-facing impact until the HTTPS storage protocol is supported in clients as well.

0
newsfragments/3802.minor → newsfragments/3904.minor
View File

0
newsfragments/3909.minor
View File

0
newsfragments/3913.minor
View File

0
newsfragments/3816.minor → newsfragments/3914.minor
View File

0
newsfragments/3915.minor
View File

0
newsfragments/3916.minor
View File

0
newsfragments/3855.minor → newsfragments/3917.minor
View File

5
newsfragments/3921.feature Normal file
View File

@ -0,0 +1,5 @@
`tahoe run ...` will now exit when its stdin is closed.
This facilitates subprocess management, specifically cleanup.
When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed.
Subsequently "tahoe run" notices this and exits.

1
newsfragments/3922.documentation Normal file
View File

@ -0,0 +1 @@
Several minor errors in the Great Black Swamp proposed specification document have been fixed.

0
newsfragments/3858.minor → newsfragments/3927.minor
View File

0
newsfragments/3859.minor → newsfragments/3928.minor
View File

0
newsfragments/3860.minor → newsfragments/3936.minor
View File

0
newsfragments/3867.minor → newsfragments/3937.minor
View File

1
newsfragments/3938.bugfix Normal file
View File

@ -0,0 +1 @@
Work with (and require) newer versions of pycddl.

1
newsfragments/3939.bugfix Normal file
View File

@ -0,0 +1 @@
Uploading immutables will now better use available bandwidth, which should allow for faster uploads in many cases.

0
newsfragments/3868.minor → newsfragments/3940.minor
View File

0
newsfragments/3526.minor → newsfragments/3942.minor
View File

0
newsfragments/3871.minor → newsfragments/3944.minor
View File

1
newsfragments/3946.bugfix Normal file
View File

@ -0,0 +1 @@
Downloads of large immutables should now finish much faster.

0
newsfragments/3872.minor → newsfragments/3947.minor
View File

0
newsfragments/3875.minor → newsfragments/3950.minor
View File

0
newsfragments/3876.minor → newsfragments/3952.minor
View File

0
newsfragments/3877.minor → newsfragments/3953.minor
View File

0
newsfragments/3881.minor → newsfragments/3954.minor
View File

0
newsfragments/3882.minor → newsfragments/3956.minor
View File

0
newsfragments/3883.minor → newsfragments/3958.minor
View File

0
newsfragments/3889.minor → newsfragments/3959.minor
View File

0
newsfragments/3890.minor → newsfragments/3960.minor
View File

1
newsfragments/3961.other Normal file
View File

@ -0,0 +1 @@
The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations.

1
newsfragments/3962.feature Normal file
View File

@ -0,0 +1 @@
Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties.

1
newsfragments/3964.removed Normal file
View File

@ -0,0 +1 @@
Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested.

0
newsfragments/3891.minor → newsfragments/3965.minor
View File

1
newsfragments/3966.bugfix Normal file
View File

@ -0,0 +1 @@
Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller.

0
newsfragments/3893.minor → newsfragments/3967.minor
View File

0
newsfragments/3895.minor → newsfragments/3968.minor
View File

0
newsfragments/3896.minor → newsfragments/3969.minor
View File

1
newsfragments/3971.minor Normal file
View File

@ -0,0 +1 @@
Changes made to mypy.ini to make mypy more 'strict' and prevent future regressions.

0
newsfragments/3898.minor → newsfragments/3974.minor
View File

1
newsfragments/3975.minor Normal file
View File

@ -0,0 +1 @@
Fixes truthy conditional in status.py

1
newsfragments/3976.minor Normal file
View File

@ -0,0 +1 @@
Fixes variable name same as built-in type.

1
newsfragments/3982.feature Normal file
View File

@ -0,0 +1 @@
Added support for Python 3.11.

0
newsfragments/3900.minor → newsfragments/3987.minor
View File

51
nix/pycddl.nix Normal file
View File

@ -0,0 +1,51 @@
# package https://gitlab.com/tahoe-lafs/pycddl
#
# also in the process of being pushed upstream
# https://github.com/NixOS/nixpkgs/pull/221220
#
# we should switch to the upstream package when it is available from our
# minimum version of nixpkgs.
#
# if you need to update this package to a new pycddl release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`.
#
# 6. run `nix-build`
#
# 7. there will be an error about a hash mismatch. change the value given to
# `cargoDeps` for `hash` to the "actual" hash value report.
#
# 8. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
buildPythonPackage rec {
pname = "pycddl";
version = "0.4.0";
format = "pyproject";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
};
nativeBuildInputs = with rustPlatform; [
maturinBuildHook
cargoSetupHook
];
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs=";
};
}

50
nix/sources.json
View File

@ -1,16 +1,4 @@
{
"mach-nix": {
"branch": "master",
"description": "Create highly reproducible python environments",
"homepage": "",
"owner": "davhau",
"repo": "mach-nix",
"rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b",
"sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf",
"type": "tarball",
"url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"niv": {
"branch": "master",
"description": "Easy dependency management for Nix projects",
@ -23,40 +11,28 @@
"url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-21.05": {
"branch": "nixos-21.05",
"nixpkgs-22.11": {
"branch": "nixos-22.11",
"description": "Nix Packages collection",
"homepage": "",
"owner": "NixOS",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
"sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
"rev": "970402e6147c49603f4d06defe44d27fe51884ce",
"sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
"url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-21.11": {
"branch": "nixos-21.11",
"description": "Nix Packages collection",
"homepage": "",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "838eefb4f93f2306d4614aafb9b2375f315d917f",
"sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"pypi-deps-db": {
"nixpkgs-unstable": {
"branch": "master",
"description": "Probably the most complete python dependency database",
"description": "Nix Packages collection",
"homepage": "",
"owner": "DavHau",
"repo": "pypi-deps-db",
"rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a",
"sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "d0c9a536331227ab883b4f6964be638fa436d81f",
"sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1",
"type": "tarball",
"url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz",
"url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}

84
nix/tahoe-lafs.nix Normal file
View File

@ -0,0 +1,84 @@
{ lib
, pythonPackages
, buildPythonPackage
, tahoe-lafs-src
, extrasNames
# control how the test suite is run
, doCheck
}:
let
pname = "tahoe-lafs";
version = "1.18.0.post1";
pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras;
pythonExtraDependencies = with pythonPackages; {
tor = [ txtorcon ];
i2p = [ txi2p ];
};
pythonPackageDependencies = with pythonPackages; [
attrs
autobahn
cbor2
click
collections-extended
cryptography
distro
eliot
filelock
foolscap
future
klein
magic-wormhole
netifaces
psutil
pycddl
pyrsistent
pyutil
six
treq
twisted
# Get the dependencies for the Twisted extras we depend on, too.
twisted.passthru.optional-dependencies.tls
twisted.passthru.optional-dependencies.conch
werkzeug
zfec
zope_interface
] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
pythonCheckDependencies = with pythonPackages; [
beautifulsoup4
fixtures
hypothesis
mock
paramiko
prometheus-client
pytest
pytest-timeout
pytest-twisted
tenacity
testtools
towncrier
];
in
buildPythonPackage {
inherit pname version;
src = tahoe-lafs-src;
propagatedBuildInputs = pythonPackageDependencies;
inherit doCheck;
checkInputs = pythonCheckDependencies;
checkPhase = ''
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
python -m twisted.trial -j $NIX_BUILD_CORES allmydata
'';
meta = with lib; {
homepage = "https://tahoe-lafs.org/";
description = "secure, decentralized, fault-tolerant file store";
# Also TGPPL
license = licenses.gpl2Plus;
};
}

4
nix/tests.nix Normal file
View File

@ -0,0 +1,4 @@
# Build the package with the test suite enabled.
args@{...}: (import ../. args).override {
doCheck = true;
}

39
nix/txi2p.nix Normal file
View File

@ -0,0 +1,39 @@
# package https://github.com/tahoe-lafs/txi2p
#
# if you need to update this package to a new txi2p release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. if there are new runtime dependencies then add them to the argument list
# at the top. if there are new test dependencies add them to the
# `checkInputs` list.
#
# 6. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ fetchPypi
, buildPythonPackage
, parsley
, twisted
, unittestCheckHook
}:
buildPythonPackage rec {
pname = "txi2p-tahoe";
version = "0.3.7";
src = fetchPypi {
inherit pname version;
hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY=";
};
propagatedBuildInputs = [ twisted parsley ];
checkInputs = [ unittestCheckHook ];
pythonImportsCheck = [ "parsley" "ometa"];
}

1
pyinstaller.spec
View File

@ -36,6 +36,7 @@ hidden_imports = [
'allmydata.stats',
'base64',
'cffi',
'charset_normalizer.md__mypyc',
'collections',
'commands',
'Crypto',

3
pytest.ini Normal file
View File

@ -0,0 +1,3 @@
[pytest]
markers =
slow: marks tests as slow (not run by default; run them with '--runslow')

29
relnotes.txt
View File

@ -1,6 +1,6 @@
ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1
ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0
The Tahoe-LAFS team is pleased to announce version 1.17.1 of
The Tahoe-LAFS team is pleased to announce version 1.18.0 of
Tahoe-LAFS, an extremely reliable decentralized storage
system. Get it with "pip install tahoe-lafs", or download a
tarball here:
@ -15,10 +15,12 @@ unique security and fault-tolerance properties:
https://tahoe-lafs.readthedocs.org/en/latest/about.html
The previous stable release of Tahoe-LAFS was v1.17.0, released on
December 6, 2021.
The previous stable release of Tahoe-LAFS was v1.17.1, released on
January 7, 2022.
This release fixes two Python3-releated regressions and 4 minor bugs.
This release drops support for Python 2 and for Python 3.6 and earlier.
twistd.pid is no longer used (in favour of one with pid + process creation time).
A collection of minor bugs and issues were also fixed.
Please see ``NEWS.rst`` [1] for a complete list of changes.
@ -132,24 +134,23 @@ Of Fame" [13].
ACKNOWLEDGEMENTS
This is the nineteenth release of Tahoe-LAFS to be created
solely as a labor of love by volunteers. Thank you very much
to the team of "hackers in the public interest" who make
Tahoe-LAFS possible.
This is the twentieth release of Tahoe-LAFS to be created solely as a
labor of love by volunteers. Thank you very much to the team of
"hackers in the public interest" who make Tahoe-LAFS possible.
meejah
on behalf of the Tahoe-LAFS team
January 7, 2022
October 1, 2022
Planet Earth
[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst
[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst
[2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst
[3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects
[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL
[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst
[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html
[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL
[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst
[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html
[7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev
[8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap
[9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS

29
setup.py
View File

@ -55,7 +55,9 @@ install_requires = [
# * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs
# * foolscap 0.13.2 drops i2p support completely
# * foolscap >= 21.7 is necessary for Python 3 with i2p support.
# * foolscap >= 23.3 is necessary for Python 3.11.
"foolscap >= 21.7.0",
"foolscap >= 23.3.0; python_version > '3.10'",
# * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that
# Twisted[conch] also depends on cryptography and Twisted[tls]
@ -96,7 +98,9 @@ install_requires = [
# an sftp extra in Tahoe-LAFS, there is no point in having one.
# * Twisted 19.10 introduces Site.getContentFile which we use to get
# temporary upload files placed into a per-node temporary directory.
"Twisted[tls,conch] >= 19.10.0",
# * Twisted 22.8.0 added support for coroutine-returning functions in many
# places (mainly via `maybeDeferred`)
"Twisted[tls,conch] >= 22.8.0",
"PyYAML >= 3.11",
@ -114,7 +118,7 @@ install_requires = [
"attrs >= 18.2.0",
# WebSocket library for twisted and asyncio
"autobahn < 22.4.1", # remove this when 22.4.3 is released
"autobahn",
# Support for Python 3 transition
"future >= 0.18.2",
@ -137,10 +141,17 @@ install_requires = [
"werkzeug != 2.2.0",
"treq",
"cbor2",
"pycddl",
# 0.4 adds the ability to pass in mmap() values which greatly reduces the
# amount of copying involved.
"pycddl >= 0.4",
# Command-line parsing
"click >= 7.0",
# for pid-file support
"psutil",
"filelock",
]
setup_requires = [
@ -220,7 +231,7 @@ def run_command(args, cwd=None):
use_shell = sys.platform == "win32"
try:
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell)
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+
print("Warning: unable to run %r." % (" ".join(args),))
print(e)
return None
@ -371,14 +382,18 @@ setup(name="tahoe-lafs", # also set in __init__.py
package_dir = {'':'src'},
packages=find_packages('src') + ['allmydata.test.plugins'],
classifiers=trove_classifiers,
# We support Python 3.7 or later. 3.11 is not supported yet.
python_requires=">=3.7, <3.11",
# We support Python 3.8 or later, 3.12 is untested for now
python_requires=">=3.8, <3.12",
install_requires=install_requires,
extras_require={
# Duplicate the Twisted pywin32 dependency here. See
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some
# discussion.
':sys_platform=="win32"': ["pywin32 != 226"],
"build": [
"dulwich",
"gpg",
],
"test": [
"flake8",
# Pin a specific pyflakes so we don't have different folks
@ -388,7 +403,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
"pyflakes == 2.2.0",
"coverage ~= 5.0",
"mock",
"tox",
"tox ~= 3.0",
"pytest",
"pytest-twisted",
"hypothesis >= 3.6.1",

17
src/allmydata/cli/grid_manager.py
View File

@ -4,7 +4,6 @@ A CLI for configuring a grid manager.
from typing import Optional
from datetime import (
datetime,
timedelta,
)
@ -24,6 +23,7 @@ from allmydata.grid_manager import (
create_grid_manager,
save_grid_manager,
load_grid_manager,
current_datetime_with_zone,
)
from allmydata.util import jsonbytes as json
@ -52,7 +52,7 @@ def grid_manager(ctx, config):
class Config(object):
"""
Availble to all sub-commands as Click's context.obj
Available to all sub-commands as Click's context.obj
"""
_grid_manager = None
@ -167,7 +167,7 @@ def list(ctx):
name,
str(ctx.obj.grid_manager.storage_servers[name].public_key_string(), "utf-8")))
for cert in ctx.obj.grid_manager.storage_servers[name].certificates:
delta = datetime.utcnow() - cert.expires
delta = current_datetime_with_zone() - cert.expires
click.echo("{} cert {}: ".format(blank_name, cert.index), nl=False)
if delta.total_seconds() < 0:
click.echo("valid until {} ({})".format(cert.expires, abbreviate_time(delta)))
@ -196,7 +196,7 @@ def sign(ctx, name, expiry_days):
"No storage-server called '{}' exists".format(name)
)
certificate_data = json.dumps(certificate.asdict(), indent=4)
certificate_data = json.dumps(certificate.marshal(), indent=4)
click.echo(certificate_data)
if fp is not None:
next_serial = 0
@ -205,13 +205,10 @@ def sign(ctx, name, expiry_days):
fname = "{}.cert.{}".format(name, next_serial)
try:
f = fp.child(fname).create()
except FileExistsError:
f = None
except OSError as e:
if e.errno == 17: # file exists
f = None
else:
raise click.ClickException(
"{}: {}".format(fname, e)
)
raise click.ClickException(f"{fname}: {e}")
next_serial += 1
with f:
f.write(certificate_data.encode("ascii"))

97
src/allmydata/client.py
View File

@ -1,22 +1,13 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
# Don't use future str to prevent leaking future's newbytes into foolscap, which they break.
from past.builtins import unicode as str
from __future__ import annotations
import os
import stat
import time
import weakref
from typing import Optional
from base64 import urlsafe_b64encode
from functools import partial
# On Python 2 this will be the backported package:
@ -45,6 +36,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer
from allmydata import storage_client
from allmydata.immutable.upload import Uploader
from allmydata.immutable.offloaded import Helper
from allmydata.mutable.filenode import MutableFileNode
from allmydata.introducer.client import IntroducerClient
from allmydata.util import (
hashutil, base32, pollmixin, log, idlib,
@ -62,13 +54,13 @@ from allmydata.interfaces import (
IStatsProducer,
SDMF_VERSION,
MDMF_VERSION,
DEFAULT_MAX_SEGMENT_SIZE,
DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE,
IFoolscapStoragePlugin,
IAnnounceableStorageServer,
)
from allmydata.nodemaker import NodeMaker
from allmydata.blacklist import Blacklist
from allmydata.node import _Config
KiB=1024
MiB=1024*KiB
@ -84,7 +76,8 @@ def _is_valid_section(section_name):
"""
return (
section_name.startswith("storageserver.plugins.") or
section_name.startswith("storageclient.plugins.")
section_name.startswith("storageclient.plugins.") or
section_name in ("grid_managers", "grid_manager_certificates")
)
@ -99,10 +92,10 @@ _client_config = configutil.ValidConfiguration(
"shares.happy",
"shares.needed",
"shares.total",
"shares._max_immutable_segment_size_for_testing",
"storage.plugins",
"force_foolscap",
),
"grid_managers": None, # means "any options valid"
"grid_manager_certificates": None,
"storage": (
"debug_discard",
"enabled",
@ -119,6 +112,7 @@ _client_config = configutil.ValidConfiguration(
"storage_dir",
"plugins",
"grid_management",
"force_foolscap",
),
"sftpd": (
"accounts.file",
@ -472,7 +466,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
return introducer_clients
def create_storage_farm_broker(config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
"""
Create a StorageFarmBroker object, for use by Uploader/Downloader
(and everybody else who wants to use storage servers)
@ -599,6 +593,10 @@ def anonymous_storage_enabled(config):
@implementer(IStatsProducer)
class _Client(node.Node, pollmixin.PollMixin):
"""
This class should be refactored; see
https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931
"""
STOREDIR = 'storage'
NODETYPE = "client"
@ -616,7 +614,7 @@ class _Client(node.Node, pollmixin.PollMixin):
DEFAULT_ENCODING_PARAMETERS = {"k": 3,
"happy": 7,
"n": 10,
"max_segment_size": DEFAULT_MAX_SEGMENT_SIZE,
"max_segment_size": DEFAULT_IMMUTABLE_MAX_SEGMENT_SIZE,
}
def __init__(self, config, main_tub, i2p_provider, tor_provider, introducer_clients,
@ -666,6 +664,14 @@ class _Client(node.Node, pollmixin.PollMixin):
if webport:
self.init_web(webport) # strports string
# TODO this may be the wrong location for now? but as temporary measure
# it allows us to get NURLs for testing in test_istorageserver.py. This
# will eventually get fixed one way or another in
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3901. See also
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3931 for the bigger
# picture issue.
self.storage_nurls : Optional[set] = None
def init_stats_provider(self):
self.stats_provider = StatsProvider(self)
self.stats_provider.setServiceParent(self)
@ -828,6 +834,11 @@ class _Client(node.Node, pollmixin.PollMixin):
if anonymous_storage_enabled(self.config):
furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file)
(_, _, swissnum) = decode_furl(furl)
if hasattr(self.tub.negotiationClass, "add_storage_server"):
nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii"))
self.storage_nurls = nurls
announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls]
announcement["anonymous-storage-FURL"] = furl
enabled_storage_servers = self._enable_storage_servers(
@ -849,11 +860,9 @@ class _Client(node.Node, pollmixin.PollMixin):
grid_manager_certificates = self.config.get_grid_manager_certificates()
announcement[u"grid-manager-certificates"] = grid_manager_certificates
# XXX we should probably verify that the certificates are
# valid and not expired, as that could be confusing for the
# storage-server operator -- but then we need the public key
# of the Grid Manager (should that go in the config too,
# then? How to handle multiple grid-managers?)
# Note: certificates are not verified for validity here, but
# that may be useful. See:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3977
for ic in self.introducer_clients:
ic.publish("storage", announcement, self._node_private_key)
@ -905,6 +914,13 @@ class _Client(node.Node, pollmixin.PollMixin):
DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))
DEP["happy"] = int(self.config.get_config("client", "shares.happy", DEP["happy"]))
# At the moment this is only used for testing, thus the janky config
# attribute name.
DEP["max_segment_size"] = int(self.config.get_config(
"client",
"shares._max_immutable_segment_size_for_testing",
DEP["max_segment_size"])
)
# for the CLI to authenticate to local JSON endpoints
self._create_auth_token()
@ -1096,9 +1112,40 @@ class _Client(node.Node, pollmixin.PollMixin):
def create_immutable_dirnode(self, children, convergence=None):
return self.nodemaker.create_immutable_directory(children, convergence)
def create_mutable_file(self, contents=None, version=None):
def create_mutable_file(
self,
contents: bytes | None = None,
version: int | None = None,
*,
unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
) -> MutableFileNode:
"""
Create *and upload* a new mutable object.
:param contents: If given, the initial contents for the new object.
:param version: If given, the mutable file format for the new object
(otherwise a format will be chosen automatically).
:param unique_keypair: **Warning** This value independently determines
the identity of the mutable object to create. There cannot be two
different mutable objects that share a keypair. They will merge
into one object (with undefined contents).
It is common to pass a None value (or not pass a valuye) for this
parameter. In these cases, a new random keypair will be
generated.
If non-None, the given public/private keypair will be used for the
new object. The expected use-case is for implementing compliance
tests.
:return: A Deferred which will fire with a representation of the new
mutable object after it has been uploaded.
"""
return self.nodemaker.create_mutable_file(contents,
version=version)
version=version,
keypair=unique_keypair)
def upload(self, uploadable, reactor=None):
uploader = self.getServiceNamed("uploader")

86
src/allmydata/crypto/rsa.py
View File

@ -9,17 +9,14 @@ features of any objects that `cryptography` documents.
That is, the public and private keys are opaque objects; DO NOT depend
on any of their methods.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing_extensions import TypeAlias
from typing import Callable
from functools import partial
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
@ -30,6 +27,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l
from allmydata.crypto.error import BadSignature
PublicKey: TypeAlias = rsa.RSAPublicKey
PrivateKey: TypeAlias = rsa.RSAPrivateKey
# This is the value that was used by `pycryptopp`, and we must continue to use it for
# both backwards compatibility and interoperability.
@ -46,12 +45,12 @@ RSA_PADDING = padding.PSS(
def create_signing_keypair(key_size):
def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]:
"""
Create a new RSA signing (private) keypair from scratch. Can be used with
`sign_data` function.
:param int key_size: length of key in bits
:param key_size: length of key in bits
:returns: 2-tuple of (private_key, public_key)
"""
@ -63,32 +62,62 @@ def create_signing_keypair(key_size):
return priv_key, priv_key.public_key()
def create_signing_keypair_from_string(private_key_der):
def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]:
"""
Create an RSA signing (private) key from previously serialized
private key bytes.
:param bytes private_key_der: blob as returned from `der_string_from_signing_keypair`
:param private_key_der: blob as returned from `der_string_from_signing_keypair`
:returns: 2-tuple of (private_key, public_key)
"""
priv_key = load_der_private_key(
_load = partial(
load_der_private_key,
private_key_der,
password=None,
backend=default_backend(),
)
if not isinstance(priv_key, rsa.RSAPrivateKey):
def load_with_validation() -> PrivateKey:
k = _load()
assert isinstance(k, PrivateKey)
return k
def load_without_validation() -> PrivateKey:
k = _load(unsafe_skip_rsa_key_validation=True)
assert isinstance(k, PrivateKey)
return k
# Load it once without the potentially expensive OpenSSL validation
# checks. These have superlinear complexity. We *will* run them just
# below - but first we'll apply our own constant-time checks.
load: Callable[[], PrivateKey] = load_without_validation
try:
unsafe_priv_key = load()
except TypeError:
# cryptography<39 does not support this parameter, so just load the
# key with validation...
unsafe_priv_key = load_with_validation()
# But avoid *reloading* it since that will run the expensive
# validation *again*.
load = lambda: unsafe_priv_key
if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey):
raise ValueError(
"Private Key did not decode to an RSA key"
)
if priv_key.key_size != 2048:
if unsafe_priv_key.key_size != 2048:
raise ValueError(
"Private Key must be 2048 bits"
)
return priv_key, priv_key.public_key()
# Now re-load it with OpenSSL's validation applied.
safe_priv_key = load()
return safe_priv_key, safe_priv_key.public_key()
def der_string_from_signing_key(private_key):
def der_string_from_signing_key(private_key: PrivateKey) -> bytes:
"""
Serializes a given RSA private key to a DER string
@ -98,14 +127,14 @@ def der_string_from_signing_key(private_key):
:returns: bytes representing `private_key`
"""
_validate_private_key(private_key)
return private_key.private_bytes(
return private_key.private_bytes( # type: ignore[attr-defined]
encoding=Encoding.DER,
format=PrivateFormat.PKCS8,
encryption_algorithm=NoEncryption(),
)
def der_string_from_verifying_key(public_key):
def der_string_from_verifying_key(public_key: PublicKey) -> bytes:
"""
Serializes a given RSA public key to a DER string.
@ -121,7 +150,7 @@ def der_string_from_verifying_key(public_key):
)
def create_verifying_key_from_string(public_key_der):
def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey:
"""
Create an RSA verifying key from a previously serialized public key
@ -134,15 +163,16 @@ def create_verifying_key_from_string(public_key_der):
public_key_der,
backend=default_backend(),
)
assert isinstance(pub_key, PublicKey)
return pub_key
def sign_data(private_key, data):
def sign_data(private_key: PrivateKey, data: bytes) -> bytes:
"""
:param private_key: the private part of a keypair returned from
`create_signing_keypair_from_string` or `create_signing_keypair`
:param bytes data: the bytes to sign
:param data: the bytes to sign
:returns: bytes which are a signature of the bytes given as `data`.
"""
@ -153,7 +183,7 @@ def sign_data(private_key, data):
hashes.SHA256(),
)
def verify_signature(public_key, alleged_signature, data):
def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None:
"""
:param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key`
@ -173,23 +203,23 @@ def verify_signature(public_key, alleged_signature, data):
raise BadSignature()
def _validate_public_key(public_key):
def _validate_public_key(public_key: PublicKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(public_key, rsa.RSAPublicKey):
raise ValueError(
"public_key must be an RSAPublicKey"
f"public_key must be an RSAPublicKey not {type(public_key)}"
)
def _validate_private_key(private_key):
def _validate_private_key(private_key: PrivateKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(private_key, rsa.RSAPrivateKey):
raise ValueError(
"private_key must be an RSAPrivateKey"
f"private_key must be an RSAPrivateKey not {type(private_key)}"
)

Some files were not shown because too many files have changed in this diff Show More