ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 04:57:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into 2916.grid-manager-proposal.6

Browse Source
This commit is contained in:
meejah 2023-02-20 11:24:20 -07:00
commit 464498f6bf
107 changed files with 21194 additions and 1945 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
.circleci/circleci.txt Normal file
View File

@ -0,0 +1,78 @@
# A master build looks like this:
# BASH_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=master
# CIRCLE_BUILD_NUM=76545
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76545
# CIRCLE_JOB=NixOS 21.11
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=ed0bda2d7456f4a2cd60870072e1fe79864a49a1
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build
# CIRCLE_USERNAME=alice
# CIRCLE_WORKFLOW_ID=6d9bb71c-be3a-4659-bf27-60954180619b
# CIRCLE_WORKFLOW_JOB_ID=0793c975-7b9f-489f-909b-8349b72d2785
# CIRCLE_WORKFLOW_WORKSPACE_ID=6d9bb71c-be3a-4659-bf27-60954180619b
# CIRCLE_WORKING_DIRECTORY=~/project
# A build of an in-repo PR looks like this:
# BASH_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=3946-less-chatty-downloads
# CIRCLE_BUILD_NUM=76612
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76612
# CIRCLE_JOB=NixOS 21.11
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=921a2083dcefdb5f431cdac195fc9ac510605349
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build
# CIRCLE_USERNAME=bob
# CIRCLE_WORKFLOW_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1
# CIRCLE_WORKFLOW_JOB_ID=316ca408-81b4-4c96-bbdd-644e4c3e01e5
# CIRCLE_WORKFLOW_WORKSPACE_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1
# CIRCLE_WORKING_DIRECTORY=~/project
# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251
# A build of a PR from a fork looks like this:
# BASH_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build
# CI=true
# CIRCLECI=true
# CIRCLE_BRANCH=pull/1252
# CIRCLE_BUILD_NUM=76678
# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76678
# CIRCLE_JOB=NixOS 21.05
# CIRCLE_NODE_INDEX=0
# CIRCLE_NODE_TOTAL=1
# CIRCLE_PROJECT_REPONAME=tahoe-lafs
# CIRCLE_PROJECT_USERNAME=tahoe-lafs
# CIRCLE_PR_NUMBER=1252
# CIRCLE_PR_REPONAME=tahoe-lafs
# CIRCLE_PR_USERNAME=carol
# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git
# CIRCLE_SHA1=15c7916e0812e6baa2a931cd54b18f3382a8456e
# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build
# CIRCLE_USERNAME=
# CIRCLE_WORKFLOW_ID=19c917c8-3a38-4b20-ac10-3265259fa03e
# CIRCLE_WORKFLOW_JOB_ID=58e95215-eccf-4664-a231-1dba7fd2d323
# CIRCLE_WORKFLOW_WORKSPACE_ID=19c917c8-3a38-4b20-ac10-3265259fa03e
# CIRCLE_WORKING_DIRECTORY=~/project
# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252
# A build of a PR from a fork where the owner has enabled CircleCI looks
# the same as a build of an in-repo PR, except it runs on th owner's
# CircleCI namespace.

234
.circleci/config.yml
View File

@ -11,20 +11,60 @@
#
version: 2.1
# A template that can be shared between the two different image-building
# workflows.
.images: &IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub credentials
# available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-11": &DOCKERHUB_CONTEXT
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
parameters:
# Control whether the image-building workflow runs as part of this pipeline.
# Generally we do not want this to run because we don't need our
# dependencies to move around all the time and because building the image
# takes a couple minutes.
#
# An easy way to trigger a pipeline with this set to true is with the
# rebuild-images.sh tool in this directory. You can also do so via the
# CircleCI web UI.
build-images:
default: false
type: "boolean"
# Control whether the test-running workflow runs as part of this pipeline.
# Generally we do want this to run because running the tests is the primary
# purpose of this pipeline.
run-tests:
default: true
type: "boolean"
workflows:
ci:
when: "<< pipeline.parameters.run-tests >>"
jobs:
# Start with jobs testing various platforms.
- "debian-10":
{}
- "debian-11":
{}
- "ubuntu-20-04":
{}
- "ubuntu-18-04":
requires:
- "ubuntu-20-04"
# Equivalent to RHEL 8; CentOS 8 is dead.
- "oraclelinux-8":
@ -54,6 +94,9 @@ workflows:
{}
- "integration":
# Run even the slow integration tests here. We need the `--` to
# sneak past tox and get to pytest.
tox-args: "-- --runslow integration"
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
@ -65,41 +108,10 @@ workflows:
{}
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
# faster and takes various spurious failures out of the critical path.
triggers:
# Build once a day
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- "master"
<<: *IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub
# credentials available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-10": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "build-image-debian-11":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
# Build as part of the workflow but only if requested.
when: "<< pipeline.parameters.build-images >>"
jobs:
@ -133,10 +145,10 @@ jobs:
steps:
- "checkout"
- run:
- run: &INSTALL_TOX
name: "Install tox"
command: |
pip install --user tox
pip install --user 'tox~=3.0'
- run:
name: "Static-ish code checks"
@ -152,9 +164,7 @@ jobs:
- "checkout"
- run:
name: "Install tox"
command: |
pip install --user tox
<<: *INSTALL_TOX
- run:
name: "Make PyInstaller executable"
@ -169,12 +179,7 @@ jobs:
command: |
dist/Tahoe-LAFS/tahoe --version
debian-10: &DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:10-py3.7"
user: "nobody"
debian-11: &DEBIAN
environment: &UTF_8_ENVIRONMENT
# In general, the test suite is not allowed to fail while the job
# succeeds. But you can set this to "yes" if you want it to be
@ -186,7 +191,7 @@ jobs:
# filenames and argv).
LANG: "en_US.UTF-8"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
@ -254,15 +259,11 @@ jobs:
/tmp/venv/bin/codecov
fi
debian-11:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Restore later using PyPy3.8
# pypy27-buster:
@ -296,6 +297,14 @@ jobs:
integration:
<<: *DEBIAN
parameters:
tox-args:
description: >-
Additional arguments to pass to the tox command.
type: "string"
default: ""
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
@ -308,28 +317,15 @@ jobs:
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
# Pass on anything we got in our parameters.
TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >>"
steps:
- "checkout"
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
ubuntu-18-04: &UBUNTU_18_04
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# The default trial args include --rterrors which is incompatible with
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
ubuntu-20-04:
<<: *DEBIAN
docker:
@ -382,7 +378,7 @@ jobs:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.3.16"
image: "nixos/nix:2.10.3"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
@ -392,27 +388,21 @@ jobs:
steps:
- "run":
# The nixos/nix image does not include ssh. Install it so the
# `checkout` step will succeed. We also want cachix for
# Nix-friendly caching.
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
nix-env \
--file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
--file $NIXPKGS \
--install \
-A openssh cachix bash
-A cachix bash
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"
- run:
name: "Cachix setup"
# Record the store paths that exist before we did much. There's no
# reason to cache these, they're either in the image or have to be
# retrieved before we can use cachix to restore from cache.
command: |
cachix use "${CACHIX_NAME}"
nix path-info --all > /tmp/store-path-pre-build
- "run":
# The Nix package doesn't know how to do this part, unfortunately.
name: "Generate version"
@ -434,55 +424,26 @@ jobs:
# build a couple simple little dependencies that don't take
# advantage of multiple cores and we get a little speedup by doing
# them in parallel.
nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
source .circleci/lib.sh
cache_if_able nix-build \
--cores 3 \
--max-jobs 2 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
- "run":
name: "Test"
command: |
# Let it go somewhat wild for the test suite itself
nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
- run:
# Send any new store objects to cachix.
name: "Push to Cachix"
when: "always"
command: |
# Cribbed from
# https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
if [ -n "$CIRCLE_PR_NUMBER" ]; then
# I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
# number of the PR being built". Sorry, dear reader, you have
# guessed poorly. It is also conditionally set based on whether
# this is a PR from a fork or not.
#
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
echo "Skipping Cachix push for forked PR."
else
# If this *isn't* a build from a fork then we have the Cachix
# write key in our environment and we can push any new objects
# to Cachix.
#
# To decide what to push, we inspect the list of store objects
# that existed before and after we did most of our work. Any
# that are new after the work is probably a useful thing to have
# around so push it to the cache. We exclude all derivation
# objects (.drv files) because they're cheap to reconstruct and
# by the time you know their cache key you've already done all
# the work anyway.
#
# This shell expression for finding the objects and pushing them
# was from the Cachix docs:
#
# https://docs.cachix.org/continuous-integration-setup/circleci.html
#
# but they seem to have removed it now.
bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
fi
source .circleci/lib.sh
cache_if_able nix-build \
--cores 8 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
tests.nix
typechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -494,7 +455,7 @@ jobs:
docs:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -545,15 +506,6 @@ jobs:
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
build-image-debian-10:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "10"
PYTHON_VERSION: "3.7"
build-image-debian-11:
<<: *BUILD_IMAGE
@ -562,14 +514,6 @@ jobs:
TAG: "11"
PYTHON_VERSION: "3.9"
build-image-ubuntu-18-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
PYTHON_VERSION: "3.7"
build-image-ubuntu-20-04:
<<: *BUILD_IMAGE

119
.circleci/lib.sh Normal file
View File

@ -0,0 +1,119 @@
# Run a command, enabling cache writes to cachix if possible. The command is
# accepted as a variable number of positional arguments (like argv).
function cache_if_able() {
# Dump some info about our build environment.
describe_build
if is_cache_writeable; then
# If the cache is available we'll use it. This lets fork owners set
# up their own caching if they want.
echo "Cachix credentials present; will attempt to write to cache."
# The `cachix watch-exec ...` does our cache population. When it sees
# something added to the store (I guess) it pushes it to the named
# cache.
cachix watch-exec "${CACHIX_NAME}" -- "$@"
else
if is_cache_required; then
echo "Required credentials (CACHIX_AUTH_TOKEN) are missing."
return 1
else
echo "Cachix credentials missing; will not attempt cache writes."
"$@"
fi
fi
}
function is_cache_writeable() {
# We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo
# jobs will get this from CircleCI configuration but jobs from forks may
# not.
[ -v CACHIX_AUTH_TOKEN ]
}
function is_cache_required() {
# If we're building in tahoe-lafs/tahoe-lafs then we must use the cache.
# If we're building anything from a fork then we're allowed to not have
# the credentials.
is_upstream
}
# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs
# repository itself (and so we expect to have cache credentials available),
# failure otherwise.
#
# See circleci.txt for notes about how this determination is made.
function is_upstream() {
# CIRCLE_PROJECT_USERNAME is set to the org the build is happening for.
# If a PR targets a fork of the repo then this is set to something other
# than "tahoe-lafs".
[ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] &&
# CIRCLE_BRANCH is set to the real branch name for in-repo PRs and
# "pull/NNNN" for pull requests from forks.
#
# CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full
# URLs of the PR pages which share an underlying branch, with one of
# them ended with that same "pull/NNNN" for PRs from forks.
! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS"
}
# Return success if splitting $3 on $2 results in an array with any element
# that ends with $1, failure otherwise.
function any_element_endswith() {
suffix=$1
shift
sep=$1
shift
haystack=$1
shift
IFS="${sep}" read -r -a elements <<< "$haystack"
for elem in "${elements[@]}"; do
if endswith "$suffix" "$elem"; then
return 0
fi
done
return 1
}
# Return success if $2 ends with $1, failure otherwise.
function endswith() {
suffix=$1
shift
haystack=$1
shift
case "$haystack" in
*${suffix})
return 0
;;
*)
return 1
;;
esac
}
function describe_build() {
echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}"
echo "Building branch: ${CIRCLE_BRANCH}"
if is_upstream; then
echo "Upstream build."
else
echo "Non-upstream build."
fi
if is_cache_required; then
echo "Cache is required."
else
echo "Cache not required."
fi
if is_cache_writeable; then
echo "Cache is writeable."
else
echo "Cache not writeable."
fi
}

2
.circleci/populate-wheelhouse.sh
View File

@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel"
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
TEST_DEPS="tox~=3.0 codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.

20
.circleci/rebuild-images.sh Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
# Get your API token here:
# https://app.circleci.com/settings/user/tokens
API_TOKEN=$1
shift
# Name the branch you want to trigger the build for
BRANCH=$1
shift
curl \
--verbose \
--request POST \
--url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \
--header "Circle-Token: $API_TOKEN" \
--header "content-type: application/json" \
--data '{"branch":"'"$BRANCH"'","parameters":{"build-images":true,"run-tests":false}}'

9
.circleci/run-tests.sh
View File

@ -45,14 +45,15 @@ fi
# A prefix for the test command that ensure it will exit after no more than a
# certain amount of time. Ideally, we would only enforce a "silent" period
# timeout but there isn't obviously a ready-made tool for that. The test
# suite only takes about 5 - 6 minutes on CircleCI right now. 15 minutes
# seems like a moderately safe window.
# timeout but there isn't obviously a ready-made tool for that. The unit test
# suite only takes about 5 - 6 minutes on CircleCI right now. The integration
# tests are a bit longer than that. 45 minutes seems like a moderately safe
# window.
#
# This is primarily aimed at catching hangs on the PyPy job which runs for
# about 21 minutes and then gets killed by CircleCI in a way that fails the
# job and bypasses our "allowed failure" logic.
TIMEOUT="timeout --kill-after 1m 15m"
TIMEOUT="timeout --kill-after 1m 45m"
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable

142
.github/workflows/ci.yml vendored
View File

@ -6,6 +6,16 @@ on:
- "master"
pull_request:
# At the start of each workflow run, GitHub creates a unique
# GITHUB_TOKEN secret to use in the workflow. It is a good idea for
# this GITHUB_TOKEN to have the minimum of permissions. See:
#
# - https://docs.github.com/en/actions/security-guides/automatic-token-authentication
# - https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
#
permissions:
contents: read
# Control to what degree jobs in this workflow will run concurrently with
# other instances of themselves.
#
@ -38,73 +48,66 @@ jobs:
- windows-latest
- ubuntu-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
include:
# On macOS don't bother with 3.7-3.8, just to get faster builds.
# On macOS don't bother with 3.8, just to get faster builds.
- os: macos-latest
python-version: "3.9"
- os: macos-latest
python-version: "3.10"
# We only support PyPy on Linux at the moment.
- os: ubuntu-latest
python-version: "pypy-3.7"
- os: ubuntu-latest
python-version: "pypy-3.8"
- os: ubuntu-latest
python-version: "pypy-3.9"
steps:
# See https://github.com/actions/checkout. A fetch-depth of 0
# fetches all tags and branches.
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
# To use pip caching with GitHub Actions in an OS-independent
# manner, we need `pip cache dir` command, which became
# available since pip v20.1+. At the time of writing this,
# GitHub Actions offers pip v20.3.3 for both ubuntu-latest and
# windows-latest, and pip v20.3.1 for macos-latest.
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
# See https://github.com/actions/cache
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade codecov tox tox-gh-actions setuptools
pip install --upgrade codecov "tox<4" tox-gh-actions setuptools
pip list
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run tox for corresponding Python version
if: ${{ !contains(matrix.os, 'windows') }}
run: python -m tox
# On Windows, a non-blocking pipe might respond (when emulating Unix-y
# API) with ENOSPC to indicate buffer full. Trial doesn't handle this
# well, so it breaks test runs. To attempt to solve this, we pipe the
# output through passthrough.py that will hopefully be able to do the right
# thing by using Windows APIs.
- name: Run tox for corresponding Python version
if: ${{ contains(matrix.os, 'windows') }}
run: |
pip install twisted pywin32
python -m tox | python misc/windows-enospc/passthrough.py
- name: Upload eliot.log
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: eliot.log
path: eliot.log
- name: Upload trial log
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
with:
name: test.log
path: _trial_temp/test.log
@ -161,21 +164,22 @@ jobs:
strategy:
fail-fast: false
matrix:
os:
- windows-latest
- ubuntu-latest
python-version:
- 3.7
- 3.9
include:
# On macOS don't bother with 3.7, just to get faster builds.
- os: macos-latest
python-version: 3.9
python-version: "3.9"
force-foolscap: false
- os: windows-latest
python-version: "3.9"
force-foolscap: false
# 22.04 has some issue with Tor at the moment:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
- os: ubuntu-20.04
python-version: "3.9"
force-foolscap: false
steps:
- name: Install Tor [Ubuntu]
if: matrix.os == 'ubuntu-latest'
if: ${{ contains(matrix.os, 'ubuntu') }}
run: sudo apt install tor
# TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744.
@ -188,51 +192,51 @@ jobs:
- name: Install Tor [Windows]
if: matrix.os == 'windows-latest'
uses: crazy-max/ghaction-chocolatey@v1
uses: crazy-max/ghaction-chocolatey@v2
with:
args: install tor
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade tox
pip install --upgrade "tox<4"
pip list
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run "Python 3 integration tests"
if: "${{ !matrix.force-foolscap }}"
env:
# On macOS this is necessary to ensure unix socket paths for tor
# aren't too long. On Windows tox won't pass it through so it has no
# effect. On Linux it doesn't make a difference one way or another.
TMPDIR: "/tmp"
run: tox -e integration
run: |
tox -e integration
- name: Run "Python 3 integration tests (force Foolscap)"
if: "${{ matrix.force-foolscap }}"
env:
# On macOS this is necessary to ensure unix socket paths for tor
# aren't too long. On Windows tox won't pass it through so it has no
# effect. On Linux it doesn't make a difference one way or another.
TMPDIR: "/tmp"
run: |
tox -e integration -- --force-foolscap integration/
- name: Upload eliot.log in case of failure
uses: actions/upload-artifact@v1
uses: actions/upload-artifact@v3
if: failure()
with:
name: integration.eliot.json
@ -253,31 +257,19 @@ jobs:
steps:
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
cache: 'pip' # caching pip dependencies
- name: Install Python packages
run: |
pip install --upgrade tox
pip install --upgrade "tox<4"
pip list
- name: Display tool versions
@ -291,7 +283,7 @@ jobs:
run: dist/Tahoe-LAFS/tahoe --version
- name: Upload PyInstaller package
uses: actions/upload-artifact@v2
uses: actions/upload-artifact@v3
with:
name: Tahoe-LAFS-${{ matrix.os }}-Python-${{ matrix.python-version }}
path: dist/Tahoe-LAFS-*-*.*

10
Dockerfile
View File

@ -1,10 +0,0 @@
FROM python:2.7
ADD . /tahoe-lafs
RUN \
cd /tahoe-lafs && \
git pull --depth=100 && \
pip install . && \
rm -rf ~/.cache/
WORKDIR /root

25
Dockerfile.dev
View File

@ -1,25 +0,0 @@
FROM debian:9
LABEL maintainer "gordon@leastauthority.com"
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get -yq upgrade
RUN DEBIAN_FRONTEND=noninteractive apt-get -yq install build-essential python-dev libffi-dev libssl-dev python-virtualenv git
RUN \
git clone https://github.com/tahoe-lafs/tahoe-lafs.git /root/tahoe-lafs; \
cd /root/tahoe-lafs; \
virtualenv --python=python2.7 venv; \
./venv/bin/pip install --upgrade setuptools; \
./venv/bin/pip install --editable .; \
./venv/bin/tahoe --version;
RUN \
cd /root; \
mkdir /root/.tahoe-client; \
mkdir /root/.tahoe-introducer; \
mkdir /root/.tahoe-server;
RUN /root/tahoe-lafs/venv/bin/tahoe create-introducer --location=tcp:introducer:3458 --port=tcp:3458 /root/.tahoe-introducer
RUN /root/tahoe-lafs/venv/bin/tahoe start /root/.tahoe-introducer
RUN /root/tahoe-lafs/venv/bin/tahoe create-node --location=tcp:server:3457 --port=tcp:3457 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) /root/.tahoe-server
RUN /root/tahoe-lafs/venv/bin/tahoe create-client --webport=3456 --introducer=$(cat /root/.tahoe-introducer/private/introducer.furl) --basedir=/root/.tahoe-client --shares-needed=1 --shares-happy=1 --shares-total=1
VOLUME ["/root/.tahoe-client", "/root/.tahoe-server", "/root/.tahoe-introducer"]
EXPOSE 3456 3457 3458
ENTRYPOINT ["/root/tahoe-lafs/venv/bin/tahoe"]
CMD []

2
README.rst
View File

@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS <docs/running.rst>`__
🐍 Python 2
-----------
Python 3.7 or later is now required.
Python 3.8 or later is required.
If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1.

106
benchmarks/upload_download.py Normal file
View File

@ -0,0 +1,106 @@
"""
First attempt at benchmarking uploads and downloads.
To run:
$ pytest benchmarks/upload_download.py -s -v -Wignore
TODO Parameterization (pytest?)
- Foolscap vs not foolscap
- Number of nodes
- Data size
- Number of needed/happy/total shares.
CAVEATS: The goal here isn't a realistic benchmark, or a benchmark that will be
measured over time, or is expected to be maintainable over time. This is just
a quick and easy way to measure the speed of certain operations, compare HTTP
and Foolscap, and see the short-term impact of changes.
Eventually this will be replaced by a real benchmark suite that can be run over
time to measure something more meaningful.
"""
from time import time, process_time
from contextlib import contextmanager
from tempfile import mkdtemp
import os
from twisted.trial.unittest import TestCase
from allmydata.util.deferredutil import async_to_deferred
from allmydata.util.consumer import MemoryConsumer
from allmydata.test.common_system import SystemTestMixin
from allmydata.immutable.upload import Data as UData
from allmydata.mutable.publish import MutableData
@contextmanager
def timeit(name):
start = time()
start_cpu = process_time()
try:
yield
finally:
print(
f"{name}: {time() - start:.3f} elapsed, {process_time() - start_cpu:.3f} CPU"
)
class ImmutableBenchmarks(SystemTestMixin, TestCase):
"""Benchmarks for immutables."""
# To use Foolscap, change to True:
FORCE_FOOLSCAP_FOR_STORAGE = False
@async_to_deferred
async def setUp(self):
SystemTestMixin.setUp(self)
self.basedir = os.path.join(mkdtemp(), "nodes")
# 2 nodes
await self.set_up_nodes(2)
# 1 share
for c in self.clients:
c.encoding_params["k"] = 1
c.encoding_params["happy"] = 1
c.encoding_params["n"] = 1
print()
@async_to_deferred
async def test_upload_and_download_immutable(self):
# To test larger files, change this:
DATA = b"Some data to upload\n" * 10
for i in range(5):
# 1. Upload:
with timeit(" upload"):
uploader = self.clients[0].getServiceNamed("uploader")
results = await uploader.upload(UData(DATA, convergence=None))
# 2. Download:
with timeit("download"):
uri = results.get_uri()
node = self.clients[1].create_node_from_uri(uri)
mc = await node.read(MemoryConsumer(), 0, None)
self.assertEqual(b"".join(mc.chunks), DATA)
@async_to_deferred
async def test_upload_and_download_mutable(self):
# To test larger files, change this:
DATA = b"Some data to upload\n" * 10
for i in range(5):
# 1. Upload:
with timeit(" upload"):
result = await self.clients[0].create_mutable_file(MutableData(DATA))
# 2. Download:
with timeit("download"):
data = await result.download_best_version()
self.assertEqual(data, DATA)

2
default.nix
View File

@ -29,7 +29,7 @@ in
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
# for dependency resolution
, pythonVersion ? "python37" # a string choosing the python derivation from
, pythonVersion ? "python39" # a string choosing the python derivation from
# nixpkgs to target
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,

49
docker-compose.yml
View File

@ -1,49 +0,0 @@
version: '2'
services:
client:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3456:3456"
depends_on:
- "introducer"
- "server"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-client"]
server:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3457:3457"
depends_on:
- "introducer"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-server"]
introducer:
build:
context: .
dockerfile: ./Dockerfile.dev
volumes:
- ./misc:/root/tahoe-lafs/misc
- ./integration:/root/tahoe-lafs/integration
- ./src:/root/tahoe-lafs/static
- ./setup.cfg:/root/tahoe-lafs/setup.cfg
- ./setup.py:/root/tahoe-lafs/setup.py
ports:
- "127.0.0.1:3458:3458"
entrypoint: /root/tahoe-lafs/venv/bin/tahoe
command: ["run", "/root/.tahoe-introducer"]

53
docs/proposed/http-storage-node-protocol.rst
View File

@ -30,12 +30,12 @@ Glossary
introducer
a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers
fURL
:ref:`fURLs <fURLs>`
a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol
(the storage service is an example of such an object)
NURL
a self-authenticating URL-like string almost exactly like a NURL but without being tied to Foolscap
:ref:`NURLs <NURLs>`
a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap
swissnum
a short random string which is part of a fURL/NURL and which acts as a shared secret to authorize clients to use a storage service
@ -579,24 +579,6 @@ Responses:
the response is ``CONFLICT``.
At this point the only thing to do is abort the upload and start from scratch (see below).
``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
This cancels an *in-progress* upload.
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response code:
* When the upload is still in progress and therefore the abort has succeeded,
the response is ``OK``.
Future uploads can start from scratch with no pre-existing upload state stored on the server.
* If the uploaded has already finished, the response is 405 (Method Not Allowed)
and no change is made.
Discussion
``````````
@ -615,6 +597,25 @@ From RFC 7231::
PATCH method defined in [RFC5789]).
``PUT /storage/v1/immutable/:storage_index/:share_number/abort``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
This cancels an *in-progress* upload.
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response code:
* When the upload is still in progress and therefore the abort has succeeded,
the response is ``OK``.
Future uploads can start from scratch with no pre-existing upload state stored on the server.
* If the uploaded has already finished, the response is 405 (Method Not Allowed)
and no change is made.
``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -624,7 +625,7 @@ corruption. It also includes potentially important details about the share.
For example::
{"reason": u"expected hash abcd, got hash efgh"}
{"reason": "expected hash abcd, got hash efgh"}
.. share-type, storage-index, and share-number are inferred from the URL
@ -798,6 +799,7 @@ Immutable Data
<first 16 bytes of share data>
200 OK
{ "required": [ {"begin": 16, "end": 48 } ] }
PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
Authorization: Tahoe-LAFS nurl-swissnum
@ -806,6 +808,7 @@ Immutable Data
<second 16 bytes of share data>
200 OK
{ "required": [ {"begin": 32, "end": 48 } ] }
PATCH /storage/v1/immutable/AAAAAAAAAAAAAAAA/7
Authorization: Tahoe-LAFS nurl-swissnum
@ -822,6 +825,7 @@ Immutable Data
Range: bytes=0-47
200 OK
Content-Range: bytes 0-47/48
<complete 48 bytes of previously uploaded data>
#. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``::
@ -905,9 +909,12 @@ otherwise it will read a byte which won't match `b""`::
#. Download the contents of share number ``3``::
GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10
GET /storage/v1/mutable/BBBBBBBBBBBBBBBB?share=3
Authorization: Tahoe-LAFS nurl-swissnum
Range: bytes=0-16
200 OK
Content-Range: bytes 0-15/16
<complete 16 bytes of previously uploaded data>
#. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``::

4
docs/specifications/url.rst
View File

@ -7,6 +7,8 @@ These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to r
An attempt is also made to outline the rationale for certain choices about these URLs.
The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs.
.. _furls:
Background
----------
@ -31,6 +33,8 @@ The client's use of the swissnum is what allows the server to authorize the clie
.. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number
.. _NURLs:
NURLs
-----

38
integration/conftest.py
View File

@ -1,15 +1,6 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
import shutil
from time import sleep
@ -49,7 +40,6 @@ from .util import (
await_client_ready,
TahoeProcess,
cli,
_run_node,
generate_ssh_key,
block_with_timeout,
)
@ -66,6 +56,29 @@ def pytest_addoption(parser):
"--coverage", action="store_true", dest="coverage",
help="Collect coverage statistics",
)
parser.addoption(
"--force-foolscap", action="store_true", default=False,
dest="force_foolscap",
help=("If set, force Foolscap only for the storage protocol. " +
"Otherwise HTTP will be used.")
)
parser.addoption(
"--runslow", action="store_true", default=False,
dest="runslow",
help="If set, run tests marked as slow.",
)
def pytest_collection_modifyitems(session, config, items):
if not config.option.runslow:
# The --runslow option was not given; keep only collected items not
# marked as slow.
items[:] = [
item
for item
in items
if item.get_closest_marker("slow") is None
]
@pytest.fixture(autouse=True, scope='session')
def eliot_logging():
@ -410,10 +423,9 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
pytest_twisted.blockon(process.restart_async(reactor, request))
await_client_ready(process)
print(f"Alice pid: {process.transport.pid}")
return process

119
integration/test_vectors.py Normal file
View File

@ -0,0 +1,119 @@
"""
Verify certain results against test vectors with well-known results.
"""
from __future__ import annotations
from functools import partial
from typing import AsyncGenerator, Iterator
from itertools import starmap, product
from attrs import evolve
from pytest import mark
from pytest_twisted import ensureDeferred
from . import vectors
from .vectors import parameters
from .util import reconfigure, upload, TahoeProcess
@mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS)
def test_convergence(convergence):
"""
Convergence secrets are 16 bytes.
"""
assert isinstance(convergence, bytes), "Convergence secret must be bytes"
assert len(convergence) == 16, "Convergence secret must by 16 bytes"
@mark.slow
@mark.parametrize('case,expected', vectors.capabilities.items())
@ensureDeferred
async def test_capability(reactor, request, alice, case, expected):
"""
The capability that results from uploading certain well-known data
with certain well-known parameters results in exactly the previously
computed value.
"""
# rewrite alice's config to match params and convergence
await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence)
# upload data in the correct format
actual = upload(alice, case.fmt, case.data)
# compare the resulting cap to the expected result
assert actual == expected
@ensureDeferred
async def skiptest_generate(reactor, request, alice):
"""
This is a helper for generating the test vectors.
You can re-generate the test vectors by fixing the name of the test and
running it. Normally this test doesn't run because it ran once and we
captured its output. Other tests run against that output and we want them
to run against the results produced originally, not a possibly
ever-changing set of outputs.
"""
space = starmap(
# segment_size could be a parameter someday but it's not easy to vary
# using the Python implementation so it isn't one for now.
partial(vectors.Case, segment_size=parameters.SEGMENT_SIZE),
product(
parameters.ZFEC_PARAMS,
parameters.CONVERGENCE_SECRETS,
parameters.OBJECT_DESCRIPTIONS,
parameters.FORMATS,
),
)
iterresults = generate(reactor, request, alice, space)
results = []
async for result in iterresults:
# Accumulate the new result
results.append(result)
# Then rewrite the whole output file with the new accumulator value.
# This means that if we fail partway through, we will still have
# recorded partial results -- instead of losing them all.
vectors.save_capabilities(results)
async def generate(
reactor,
request,
alice: TahoeProcess,
cases: Iterator[vectors.Case],
) -> AsyncGenerator[[vectors.Case, str], None]:
"""
Generate all of the test vectors using the given node.
:param reactor: The reactor to use to restart the Tahoe-LAFS node when it
needs to be reconfigured.
:param request: The pytest request object to use to arrange process
cleanup.
:param format: The name of the encryption/data format to use.
:param alice: The Tahoe-LAFS node to use to generate the test vectors.
:param case: The inputs for which to generate a value.
:return: The capability for the case.
"""
# Share placement doesn't affect the resulting capability. For maximum
# reliability of this generator, be happy if we can put shares anywhere
happy = 1
for case in cases:
await reconfigure(
reactor,
request,
alice,
(happy, case.params.required, case.params.total),
case.convergence
)
# Give the format a chance to make an RSA key if it needs it.
case = evolve(case, fmt=case.fmt.customize())
cap = upload(alice, case.fmt, case.data)
yield case, cap

24
integration/test_web.py
View File

@ -7,18 +7,9 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did
at the time of testing' -- not necessarily a cohesive idea of what the
WebAPI *should* do in every situation. It's not clear the latter
exists anywhere, however.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import time
from urllib.parse import unquote as url_unquote, quote as url_quote
@ -32,6 +23,7 @@ import requests
import html5lib
from bs4 import BeautifulSoup
from pytest_twisted import ensureDeferred
def test_index(alice):
"""
@ -252,10 +244,18 @@ def test_status(alice):
assert found_download, "Failed to find the file we downloaded in the status-page"
def test_directory_deep_check(alice):
@ensureDeferred
async def test_directory_deep_check(reactor, request, alice):
"""
use deep-check and confirm the result pages work
"""
# Make sure the node is configured compatibly with expectations of this
# test.
happy = 3
required = 2
total = 4
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
# create a directory
resp = requests.post(
@ -313,7 +313,7 @@ def test_directory_deep_check(alice):
)
def check_repair_data(checkdata):
assert checkdata["healthy"] is True
assert checkdata["healthy"]
assert checkdata["count-happiness"] == 4
assert checkdata["count-good-share-hosts"] == 4
assert checkdata["count-shares-good"] == 4

281
integration/util.py
View File

@ -1,22 +1,19 @@
"""
Ported to Python 3.
General functionality useful for the implementation of integration tests.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from contextlib import contextmanager
from typing import Any
from typing_extensions import Literal
from tempfile import NamedTemporaryFile
import sys
import time
import json
from os import mkdir, environ
from os.path import exists, join
from io import StringIO, BytesIO
from functools import partial
from subprocess import check_output
from twisted.python.filepath import (
@ -26,12 +23,23 @@ from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
from twisted.internet.threads import deferToThread
from twisted.internet.interfaces import IProcessTransport, IReactorProcess
from attrs import frozen, evolve
import requests
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import (
Encoding,
PrivateFormat,
NoEncryption,
)
from paramiko.rsakey import RSAKey
from boltons.funcutils import wraps
from allmydata.util import base32
from allmydata.util.configutil import (
get_config,
set_config,
@ -142,9 +150,40 @@ class _MagicTextProtocol(ProcessProtocol):
sys.stdout.write(data)
def _cleanup_tahoe_process(tahoe_transport, exited):
def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
"""
Terminate the given process with a kill signal (SIGKILL on POSIX,
If the given process transport seems to still be associated with a
running process, send a SIGTERM to that process.
:param transport: The transport to use.
:param allow_missing: If ``True`` then it is not an error for the
transport to have no associated process. Otherwise, an exception will
be raised in that case.
:raise: ``ValueError`` if ``allow_missing`` is ``False`` and the transport
has no process.
"""
if transport.pid is None:
if allow_missing:
print("Process already cleaned up and that's okay.")
return
else:
raise ValueError("Process is not running")
print("signaling {} with TERM".format(transport.pid))
try:
transport.signalProcess('TERM')
except ProcessExitedAlready:
# The transport object thought it still had a process but the real OS
# process has already exited. That's fine. We accomplished what we
# wanted to. We don't care about ``allow_missing`` here because
# there's no way we could have known the real OS process already
# exited.
pass
def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False):
"""
Terminate the given process with a kill signal (SIGTERM on POSIX,
TerminateProcess on Windows).
:param tahoe_transport: The `IProcessTransport` representing the process.
@ -153,14 +192,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited):
:return: After the process has exited.
"""
from twisted.internet import reactor
try:
print("signaling {} with TERM".format(tahoe_transport.pid))
tahoe_transport.signalProcess('TERM')
print("signaled, blocking on exit")
block_with_timeout(exited, reactor)
print("exited, goodbye")
except ProcessExitedAlready:
pass
_cleanup_process_async(tahoe_transport, allow_missing=allow_missing)
print("signaled, blocking on exit")
block_with_timeout(exited, reactor)
print("exited, goodbye")
def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
@ -207,8 +242,33 @@ class TahoeProcess(object):
def kill(self):
"""Kill the process, block until it's done."""
print(f"TahoeProcess.kill({self.transport.pid} / {self.node_dir})")
_cleanup_tahoe_process(self.transport, self.transport.exited)
def kill_async(self):
"""
Kill the process, return a Deferred that fires when it's done.
"""
print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})")
_cleanup_process_async(self.transport, allow_missing=False)
return self.transport.exited
def restart_async(self, reactor: IReactorProcess, request: Any) -> Deferred:
"""
Stop and then re-start the associated process.
:return: A Deferred that fires after the new process is ready to
handle requests.
"""
d = self.kill_async()
d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False))
def got_new_process(proc):
# Grab the new transport since the one we had before is no longer
# valid after the stop/start cycle.
self._process_transport = proc.transport
d.addCallback(got_new_process)
return d
def __str__(self):
return "<TahoeProcess in '{}'>".format(self._node_dir)
@ -237,19 +297,17 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
)
transport.exited = protocol.exited
tahoe_process = TahoeProcess(
transport,
node_dir,
)
if finalize:
request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
request.addfinalizer(tahoe_process.kill)
# XXX abusing the Deferred; should use .when_magic_seen() pattern
def got_proto(proto):
transport._protocol = proto
return TahoeProcess(
transport,
node_dir,
)
protocol.magic_seen.addCallback(got_proto)
return protocol.magic_seen
d = protocol.magic_seen
d.addCallback(lambda ignored: tahoe_process)
return d
def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
@ -300,6 +358,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
u'log_gatherer.furl',
flog_gatherer,
)
force_foolscap = request.config.getoption("force_foolscap")
assert force_foolscap in (True, False)
set_config(
config,
'storage',
'force_foolscap',
str(force_foolscap),
)
write_config(FilePath(config_path), config)
created_d.addCallback(created)
@ -572,3 +638,158 @@ def run_in_thread(f):
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test
@frozen
class CHK:
"""
Represent the CHK encoding sufficiently to run a ``tahoe put`` command
using it.
"""
kind = "chk"
max_shares = 256
def customize(self) -> CHK:
# Nothing to do.
return self
@classmethod
def load(cls, params: None) -> CHK:
assert params is None
return cls()
def to_json(self) -> None:
return None
@contextmanager
def to_argv(self) -> None:
yield []
@frozen
class SSK:
"""
Represent the SSK encodings (SDMF and MDMF) sufficiently to run a
``tahoe put`` command using one of them.
"""
kind = "ssk"
# SDMF and MDMF encode share counts (N and k) into the share itself as an
# unsigned byte. They could have encoded (share count - 1) to fit the
# full range supported by ZFEC into the unsigned byte - but they don't.
# So 256 is inaccessible to those formats and we set the upper bound at
# 255.
max_shares = 255
name: Literal["sdmf", "mdmf"]
key: None | bytes
@classmethod
def load(cls, params: dict) -> SSK:
assert params.keys() == {"format", "mutable", "key"}
return cls(params["format"], params["key"].encode("ascii"))
def customize(self) -> SSK:
"""
Return an SSK with a newly generated random RSA key.
"""
return evolve(self, key=generate_rsa_key())
def to_json(self) -> dict[str, str]:
return {
"format": self.name,
"mutable": None,
"key": self.key.decode("ascii"),
}
@contextmanager
def to_argv(self) -> None:
with NamedTemporaryFile() as f:
f.write(self.key)
f.flush()
yield [f"--format={self.name}", "--mutable", f"--private-key-path={f.name}"]
def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str:
"""
Upload the given data to the given node.
:param alice: The node to upload to.
:param fmt: The name of the format for the upload. CHK, SDMF, or MDMF.
:param data: The data to upload.
:return: The capability for the uploaded data.
"""
with NamedTemporaryFile() as f:
f.write(data)
f.flush()
with fmt.to_argv() as fmt_argv:
argv = [alice, "put"] + fmt_argv + [f.name]
return cli(*argv).decode("utf-8").strip()
async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: None | bytes) -> None:
"""
Reconfigure a Tahoe-LAFS node with different ZFEC parameters and
convergence secret.
If the current configuration is different from the specified
configuration, the node will be restarted so it takes effect.
:param reactor: A reactor to use to restart the process.
:param request: The pytest request object to use to arrange process
cleanup.
:param node: The Tahoe-LAFS node to reconfigure.
:param params: The ``happy``, ``needed``, and ``total`` ZFEC encoding
parameters.
:param convergence: If given, the convergence secret. If not given, the
existing convergence secret will be left alone.
:return: ``None`` after the node configuration has been rewritten, the
node has been restarted, and the node is ready to provide service.
"""
happy, needed, total = params
config = node.get_config()
changed = False
cur_happy = int(config.get_config("client", "shares.happy"))
cur_needed = int(config.get_config("client", "shares.needed"))
cur_total = int(config.get_config("client", "shares.total"))
if (happy, needed, total) != (cur_happy, cur_needed, cur_total):
changed = True
config.set_config("client", "shares.happy", str(happy))
config.set_config("client", "shares.needed", str(needed))
config.set_config("client", "shares.total", str(total))
if convergence is not None:
cur_convergence = config.get_private_config("convergence").encode("ascii")
if base32.a2b(cur_convergence) != convergence:
changed = True
config.write_private_config("convergence", base32.b2a(convergence))
if changed:
# restart the node
print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
await node.restart_async(reactor, request)
print("Restarted. Waiting for ready state.")
await_client_ready(node)
print("Ready.")
else:
print("Config unchanged, not restarting.")
def generate_rsa_key() -> bytes:
"""
Generate a 2048 bit RSA key suitable for use with SSKs.
"""
return rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
).private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
)

30
integration/vectors/__init__.py Normal file
View File

@ -0,0 +1,30 @@
__all__ = [
"DATA_PATH",
"CURRENT_VERSION",
"MAX_SHARES",
"Case",
"Sample",
"SeedParam",
"encode_bytes",
"save_capabilities",
"capabilities",
]
from .vectors import (
DATA_PATH,
CURRENT_VERSION,
Case,
Sample,
SeedParam,
encode_bytes,
save_capabilities,
capabilities,
)
from .parameters import (
MAX_SHARES,
)

58
integration/vectors/model.py Normal file
View File

@ -0,0 +1,58 @@
"""
Simple data type definitions useful in the definition/verification of test
vectors.
"""
from __future__ import annotations
from attrs import frozen
# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares!
# Represent max symbolically and resolve it when we know what format we're
# dealing with.
MAX_SHARES = "max"
@frozen
class Sample:
"""
Some instructions for building a long byte string.
:ivar seed: Some bytes to repeat some times to produce the string.
:ivar length: The length of the desired byte string.
"""
seed: bytes
length: int
@frozen
class Param:
"""
Some ZFEC parameters.
"""
required: int
total: int
@frozen
class SeedParam:
"""
Some ZFEC parameters, almost.
:ivar required: The number of required shares.
:ivar total: Either the number of total shares or the constant
``MAX_SHARES`` to indicate that the total number of shares should be
the maximum number supported by the object format.
"""
required: int
total: int | str
def realize(self, max_total: int) -> Param:
"""
Create a ``Param`` from this object's values, possibly
substituting the given real value for total if necessary.
:param max_total: The value to use to replace ``MAX_SHARES`` if
necessary.
"""
if self.total == MAX_SHARES:
return Param(self.required, max_total)
return Param(self.required, self.total)

93
integration/vectors/parameters.py Normal file
View File

@ -0,0 +1,93 @@
"""
Define input parameters for test vector generation.
:ivar CONVERGENCE_SECRETS: Convergence secrets.
:ivar SEGMENT_SIZE: The single segment size that the Python implementation
currently supports without a lot of refactoring.
:ivar OBJECT_DESCRIPTIONS: Small objects with instructions which can be
expanded into a possibly large byte string. These are intended to be used
as plaintext inputs.
:ivar ZFEC_PARAMS: Input parameters to ZFEC.
:ivar FORMATS: Encoding/encryption formats.
"""
from __future__ import annotations
from hashlib import sha256
from .model import MAX_SHARES
from .vectors import Sample, SeedParam
from ..util import CHK, SSK
def digest(bs: bytes) -> bytes:
"""
Digest bytes to bytes.
"""
return sha256(bs).digest()
def hexdigest(bs: bytes) -> str:
"""
Digest bytes to text.
"""
return sha256(bs).hexdigest()
# Just a couple convergence secrets. The only thing we do with this value is
# feed it into a tagged hash. It certainly makes a difference to the output
# but the hash should destroy any structure in the input so it doesn't seem
# like there's a reason to test a lot of different values.
CONVERGENCE_SECRETS: list[bytes] = [
b"aaaaaaaaaaaaaaaa",
digest(b"Hello world")[:16],
]
SEGMENT_SIZE: int = 128 * 1024
# Exercise at least a handful of different sizes, trying to cover:
#
# 1. Some cases smaller than one "segment" (128k).
# This covers shrinking of some parameters to match data size.
# This includes one case of the smallest possible CHK.
#
# 2. Some cases right on the edges of integer segment multiples.
# Because boundaries are tricky.
#
# 4. Some cases that involve quite a few segments.
# This exercises merkle tree construction more thoroughly.
#
# See ``stretch`` for construction of the actual test data.
OBJECT_DESCRIPTIONS: list[Sample] = [
# The smallest possible. 55 bytes and smaller are LIT.
Sample(b"a", 56),
Sample(b"a", 1024),
Sample(b"c", 4096),
Sample(digest(b"foo"), SEGMENT_SIZE - 1),
Sample(digest(b"bar"), SEGMENT_SIZE + 1),
Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1),
Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1),
Sample(digest(b"bazquux"), SEGMENT_SIZE * 32),
Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1),
Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1),
]
ZFEC_PARAMS: list[SeedParam] = [
SeedParam(1, 1),
SeedParam(1, 3),
SeedParam(2, 3),
SeedParam(3, 10),
SeedParam(71, 255),
SeedParam(101, MAX_SHARES),
]
FORMATS: list[CHK | SSK] = [
CHK(),
# These start out unaware of a key but various keys will be supplied
# during generation.
SSK(name="sdmf", key=None),
SSK(name="mdmf", key=None),
]

18002
integration/vectors/test_vectors.yaml Executable file
View File

File diff suppressed because it is too large Load Diff

155
integration/vectors/vectors.py Normal file
View File

@ -0,0 +1,155 @@
"""
A module that loads pre-generated test vectors.
:ivar DATA_PATH: The path of the file containing test vectors.
:ivar capabilities: The capability test vectors.
"""
from __future__ import annotations
from typing import TextIO
from attrs import frozen
from yaml import safe_load, safe_dump
from base64 import b64encode, b64decode
from twisted.python.filepath import FilePath
from .model import Param, Sample, SeedParam
from ..util import CHK, SSK
DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml")
# The version of the persisted test vector data this code can interpret.
CURRENT_VERSION: str = "2023-01-16.2"
@frozen
class Case:
"""
Represent one case for which we want/have a test vector.
"""
seed_params: Param
convergence: bytes
seed_data: Sample
fmt: CHK | SSK
segment_size: int
@property
def data(self):
return stretch(self.seed_data.seed, self.seed_data.length)
@property
def params(self):
return self.seed_params.realize(self.fmt.max_shares)
def encode_bytes(b: bytes) -> str:
"""
Base64 encode some bytes to text so they are representable in JSON.
"""
return b64encode(b).decode("ascii")
def decode_bytes(b: str) -> bytes:
"""
Base64 decode some text to bytes.
"""
return b64decode(b.encode("ascii"))
def stretch(seed: bytes, size: int) -> bytes:
"""
Given a simple description of a byte string, return the byte string
itself.
"""
assert isinstance(seed, bytes)
assert isinstance(size, int)
assert size > 0
assert len(seed) > 0
multiples = size // len(seed) + 1
return (seed * multiples)[:size]
def save_capabilities(results: list[tuple[Case, str]], path: FilePath = DATA_PATH) -> None:
"""
Save some test vector cases and their expected values.
This is logically the inverse of ``load_capabilities``.
"""
path.setContent(safe_dump({
"version": CURRENT_VERSION,
"vector": [
{
"convergence": encode_bytes(case.convergence),
"format": {
"kind": case.fmt.kind,
"params": case.fmt.to_json(),
},
"sample": {
"seed": encode_bytes(case.seed_data.seed),
"length": case.seed_data.length,
},
"zfec": {
"segmentSize": case.segment_size,
"required": case.params.required,
"total": case.params.total,
},
"expected": cap,
}
for (case, cap)
in results
],
}).encode("ascii"))
def load_format(serialized: dict) -> CHK | SSK:
"""
Load an encrypted object format from a simple description of it.
:param serialized: A ``dict`` describing either CHK or SSK, possibly with
some parameters.
"""
if serialized["kind"] == "chk":
return CHK.load(serialized["params"])
elif serialized["kind"] == "ssk":
return SSK.load(serialized["params"])
else:
raise ValueError(f"Unrecognized format: {serialized}")
def load_capabilities(f: TextIO) -> dict[Case, str]:
"""
Load some test vector cases and their expected results from the given
file.
This is logically the inverse of ``save_capabilities``.
"""
data = safe_load(f)
if data is None:
return {}
if data["version"] != CURRENT_VERSION:
print(
f"Current version is {CURRENT_VERSION}; "
f"cannot load version {data['version']} data."
)
return {}
return {
Case(
seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]),
segment_size=case["zfec"]["segmentSize"],
convergence=decode_bytes(case["convergence"]),
seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]),
fmt=load_format(case["format"]),
): case["expected"]
for case
in data["vector"]
}
try:
with DATA_PATH.open() as f:
capabilities: dict[Case, str] = load_capabilities(f)
except FileNotFoundError:
capabilities = {}

36
misc/windows-enospc/passthrough.py Normal file
View File

@ -0,0 +1,36 @@
"""
Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on
Windows. So, this program passes through data from stdin to stdout, using
Windows APIs instead of Unix-y APIs.
"""
from twisted.internet.stdio import StandardIO
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.interfaces import IHalfCloseableProtocol
from twisted.internet.error import ReactorNotRunning
from zope.interface import implementer
@implementer(IHalfCloseableProtocol)
class Passthrough(Protocol):
def readConnectionLost(self):
self.transport.loseConnection()
def writeConnectionLost(self):
try:
reactor.stop()
except ReactorNotRunning:
pass
def dataReceived(self, data):
self.transport.write(data)
def connectionLost(self, reason):
try:
reactor.stop()
except ReactorNotRunning:
pass
std = StandardIO(Passthrough())
reactor.run()

7
mypy.ini
View File

@ -1,3 +1,10 @@
[mypy]
ignore_missing_imports = True
plugins=mypy_zope:plugin
show_column_numbers = True
pretty = True
show_error_codes = True
warn_unused_configs =True
no_implicit_optional = True
warn_redundant_casts = True
strict_equality = True

0
newsfragments/3783.minor Normal file
View File

0
newsfragments/3870.minor Normal file
View File

0
newsfragments/3874.minor Normal file
View File

0
newsfragments/3914.minor Normal file
View File

5
newsfragments/3921.feature Normal file
View File

@ -0,0 +1,5 @@
`tahoe run ...` will now exit when its stdin is closed.
This facilitates subprocess management, specifically cleanup.
When a parent process is running tahoe and exits without time to do "proper" cleanup at least the stdin descriptor will be closed.
Subsequently "tahoe run" notices this and exits.

1
newsfragments/3922.documentation Normal file
View File

@ -0,0 +1 @@
Several minor errors in the Great Black Swamp proposed specification document have been fixed.

0
newsfragments/3927.minor Normal file
View File

0
newsfragments/3937.minor Normal file
View File

1
newsfragments/3938.bugfix Normal file
View File

@ -0,0 +1 @@
Work with (and require) newer versions of pycddl.

1
newsfragments/3939.bugfix Normal file
View File

@ -0,0 +1 @@
Uploading immutables will now better use available bandwidth, which should allow for faster uploads in many cases.

0
newsfragments/3940.minor Normal file
View File

1
newsfragments/3942.minor Normal file
View File

@ -0,0 +1 @@

0
newsfragments/3944.minor Normal file
View File

0
newsfragments/3947.minor Normal file
View File

0
newsfragments/3950.minor Normal file
View File

0
newsfragments/3952.minor Normal file
View File

0
newsfragments/3953.minor Normal file
View File

0
newsfragments/3954.minor Normal file
View File

0
newsfragments/3956.minor Normal file
View File

0
newsfragments/3958.minor Normal file
View File

0
newsfragments/3960.minor Normal file
View File

1
newsfragments/3961.other Normal file
View File

@ -0,0 +1 @@
The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations.

1
newsfragments/3962.feature Normal file
View File

@ -0,0 +1 @@
Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties.

1
newsfragments/3964.removed Normal file
View File

@ -0,0 +1 @@
Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested.

1
newsfragments/3966.bugfix Normal file
View File

@ -0,0 +1 @@
Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller.

0
newsfragments/3967.minor Normal file
View File

0
newsfragments/3969.minor Normal file
View File

1
newsfragments/3971.minor Normal file
View File

@ -0,0 +1 @@
Changes made to mypy.ini to make mypy more 'strict' and prevent future regressions.

0
newsfragments/3974.minor Normal file
View File

1
newsfragments/3975.minor Normal file
View File

@ -0,0 +1 @@
Fixes truthy conditional in status.py

1
newsfragments/3976.minor Normal file
View File

@ -0,0 +1 @@
Fixes variable name same as built-in type.

16
nix/sources.json
View File

@ -1,14 +1,14 @@
{
"mach-nix": {
"branch": "master",
"branch": "switch-to-nix-pypi-fetcher-2",
"description": "Create highly reproducible python environments",
"homepage": "",
"owner": "davhau",
"owner": "PrivateStorageio",
"repo": "mach-nix",
"rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b",
"sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf",
"rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4",
"sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls",
"type": "tarball",
"url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz",
"url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"niv": {
@ -53,10 +53,10 @@
"homepage": "",
"owner": "DavHau",
"repo": "pypi-deps-db",
"rev": "76b8f1e44a8ec051b853494bcf3cc8453a294a6a",
"sha256": "18fgqyh4z578jjhk26n1xi2cw2l98vrqp962rgz9a6wa5yh1nm4x",
"rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6",
"sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n",
"type": "tarball",
"url": "https://github.com/DavHau/pypi-deps-db/archive/76b8f1e44a8ec051b853494bcf3cc8453a294a6a.tar.gz",
"url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}

1
pyinstaller.spec
View File

@ -36,6 +36,7 @@ hidden_imports = [
'allmydata.stats',
'base64',
'cffi',
'charset_normalizer.md__mypyc',
'collections',
'commands',
'Crypto',

3
pytest.ini Normal file
View File

@ -0,0 +1,3 @@
[pytest]
markers =
slow: marks tests as slow (not run by default; run them with '--runslow')

29
relnotes.txt
View File

@ -1,6 +1,6 @@
ANNOUNCING Tahoe, the Least-Authority File Store, v1.17.1
ANNOUNCING Tahoe, the Least-Authority File Store, v1.18.0
The Tahoe-LAFS team is pleased to announce version 1.17.1 of
The Tahoe-LAFS team is pleased to announce version 1.18.0 of
Tahoe-LAFS, an extremely reliable decentralized storage
system. Get it with "pip install tahoe-lafs", or download a
tarball here:
@ -15,10 +15,12 @@ unique security and fault-tolerance properties:
https://tahoe-lafs.readthedocs.org/en/latest/about.html
The previous stable release of Tahoe-LAFS was v1.17.0, released on
December 6, 2021.
The previous stable release of Tahoe-LAFS was v1.17.1, released on
January 7, 2022.
This release fixes two Python3-releated regressions and 4 minor bugs.
This release drops support for Python 2 and for Python 3.6 and earlier.
twistd.pid is no longer used (in favour of one with pid + process creation time).
A collection of minor bugs and issues were also fixed.
Please see ``NEWS.rst`` [1] for a complete list of changes.
@ -132,24 +134,23 @@ Of Fame" [13].
ACKNOWLEDGEMENTS
This is the nineteenth release of Tahoe-LAFS to be created
solely as a labor of love by volunteers. Thank you very much
to the team of "hackers in the public interest" who make
Tahoe-LAFS possible.
This is the twentieth release of Tahoe-LAFS to be created solely as a
labor of love by volunteers. Thank you very much to the team of
"hackers in the public interest" who make Tahoe-LAFS possible.
meejah
on behalf of the Tahoe-LAFS team
January 7, 2022
October 1, 2022
Planet Earth
[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/NEWS.rst
[1] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/NEWS.rst
[2] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/known_issues.rst
[3] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/RelatedProjects
[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.GPL
[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.17.1/COPYING.TGPPL.rst
[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.17.1/INSTALL.html
[4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.GPL
[5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.18.0/COPYING.TGPPL.rst
[6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.18.0/INSTALL.html
[7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev
[8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap
[9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS

20
setup.py
View File

@ -96,7 +96,9 @@ install_requires = [
# an sftp extra in Tahoe-LAFS, there is no point in having one.
# * Twisted 19.10 introduces Site.getContentFile which we use to get
# temporary upload files placed into a per-node temporary directory.
"Twisted[tls,conch] >= 19.10.0",
# * Twisted 22.8.0 added support for coroutine-returning functions in many
# places (mainly via `maybeDeferred`)
"Twisted[tls,conch] >= 22.8.0",
"PyYAML >= 3.11",
@ -137,7 +139,10 @@ install_requires = [
"werkzeug != 2.2.0",
"treq",
"cbor2",
"pycddl",
# 0.4 adds the ability to pass in mmap() values which greatly reduces the
# amount of copying involved.
"pycddl >= 0.4",
# Command-line parsing
"click >= 7.0",
@ -224,7 +229,7 @@ def run_command(args, cwd=None):
use_shell = sys.platform == "win32"
try:
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell)
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+
print("Warning: unable to run %r." % (" ".join(args),))
print(e)
return None
@ -375,8 +380,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
package_dir = {'':'src'},
packages=find_packages('src') + ['allmydata.test.plugins'],
classifiers=trove_classifiers,
# We support Python 3.7 or later. 3.11 is not supported yet.
python_requires=">=3.7, <3.11",
# We support Python 3.8 or later. 3.11 is not supported yet.
python_requires=">=3.8, <3.11",
install_requires=install_requires,
extras_require={
# Duplicate the Twisted pywin32 dependency here. See
@ -389,9 +394,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
],
"test": [
"flake8",
# On Python 3.7, importlib_metadata v5 breaks flake8.
# https://github.com/python/importlib_metadata/issues/407
"importlib_metadata<5; python_version < '3.8'",
# Pin a specific pyflakes so we don't have different folks
# disagreeing on what is or is not a lint issue. We can bump
# this version from time to time, but we will do it
@ -399,7 +401,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
"pyflakes == 2.2.0",
"coverage ~= 5.0",
"mock",
"tox",
"tox ~= 3.0",
"pytest",
"pytest-twisted",
"hypothesis >= 3.6.1",

44
src/allmydata/client.py
View File

@ -36,6 +36,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer
from allmydata import storage_client
from allmydata.immutable.upload import Uploader
from allmydata.immutable.offloaded import Helper
from allmydata.mutable.filenode import MutableFileNode
from allmydata.introducer.client import IntroducerClient
from allmydata.util import (
hashutil, base32, pollmixin, log, idlib,
@ -110,6 +111,7 @@ _client_config = configutil.ValidConfiguration(
"storage_dir",
"plugins",
"grid_management",
"force_foolscap",
),
"sftpd": (
"accounts.file",
@ -832,9 +834,10 @@ class _Client(node.Node, pollmixin.PollMixin):
furl_file = self.config.get_private_path("storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(FoolscapStorageServer(ss), furlFile=furl_file)
(_, _, swissnum) = decode_furl(furl)
self.storage_nurls = self.tub.negotiationClass.add_storage_server(
ss, swissnum.encode("ascii")
)
if hasattr(self.tub.negotiationClass, "add_storage_server"):
nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii"))
self.storage_nurls = nurls
announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls]
announcement["anonymous-storage-FURL"] = furl
enabled_storage_servers = self._enable_storage_servers(
@ -1103,9 +1106,40 @@ class _Client(node.Node, pollmixin.PollMixin):
def create_immutable_dirnode(self, children, convergence=None):
return self.nodemaker.create_immutable_directory(children, convergence)
def create_mutable_file(self, contents=None, version=None):
def create_mutable_file(
self,
contents: bytes | None = None,
version: int | None = None,
*,
unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
) -> MutableFileNode:
"""
Create *and upload* a new mutable object.
:param contents: If given, the initial contents for the new object.
:param version: If given, the mutable file format for the new object
(otherwise a format will be chosen automatically).
:param unique_keypair: **Warning** This value independently determines
the identity of the mutable object to create. There cannot be two
different mutable objects that share a keypair. They will merge
into one object (with undefined contents).
It is common to pass a None value (or not pass a valuye) for this
parameter. In these cases, a new random keypair will be
generated.
If non-None, the given public/private keypair will be used for the
new object. The expected use-case is for implementing compliance
tests.
:return: A Deferred which will fire with a representation of the new
mutable object after it has been uploaded.
"""
return self.nodemaker.create_mutable_file(contents,
version=version)
version=version,
keypair=unique_keypair)
def upload(self, uploadable, reactor=None):
uploader = self.getServiceNamed("uploader")

86
src/allmydata/crypto/rsa.py
View File

@ -9,17 +9,14 @@ features of any objects that `cryptography` documents.
That is, the public and private keys are opaque objects; DO NOT depend
on any of their methods.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing_extensions import TypeAlias
from typing import Callable
from functools import partial
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
@ -30,6 +27,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l
from allmydata.crypto.error import BadSignature
PublicKey: TypeAlias = rsa.RSAPublicKey
PrivateKey: TypeAlias = rsa.RSAPrivateKey
# This is the value that was used by `pycryptopp`, and we must continue to use it for
# both backwards compatibility and interoperability.
@ -46,12 +45,12 @@ RSA_PADDING = padding.PSS(
def create_signing_keypair(key_size):
def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]:
"""
Create a new RSA signing (private) keypair from scratch. Can be used with
`sign_data` function.
:param int key_size: length of key in bits
:param key_size: length of key in bits
:returns: 2-tuple of (private_key, public_key)
"""
@ -63,32 +62,62 @@ def create_signing_keypair(key_size):
return priv_key, priv_key.public_key()
def create_signing_keypair_from_string(private_key_der):
def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]:
"""
Create an RSA signing (private) key from previously serialized
private key bytes.
:param bytes private_key_der: blob as returned from `der_string_from_signing_keypair`
:param private_key_der: blob as returned from `der_string_from_signing_keypair`
:returns: 2-tuple of (private_key, public_key)
"""
priv_key = load_der_private_key(
_load = partial(
load_der_private_key,
private_key_der,
password=None,
backend=default_backend(),
)
if not isinstance(priv_key, rsa.RSAPrivateKey):
def load_with_validation() -> PrivateKey:
k = _load()
assert isinstance(k, PrivateKey)
return k
def load_without_validation() -> PrivateKey:
k = _load(unsafe_skip_rsa_key_validation=True)
assert isinstance(k, PrivateKey)
return k
# Load it once without the potentially expensive OpenSSL validation
# checks. These have superlinear complexity. We *will* run them just
# below - but first we'll apply our own constant-time checks.
load: Callable[[], PrivateKey] = load_without_validation
try:
unsafe_priv_key = load()
except TypeError:
# cryptography<39 does not support this parameter, so just load the
# key with validation...
unsafe_priv_key = load_with_validation()
# But avoid *reloading* it since that will run the expensive
# validation *again*.
load = lambda: unsafe_priv_key
if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey):
raise ValueError(
"Private Key did not decode to an RSA key"
)
if priv_key.key_size != 2048:
if unsafe_priv_key.key_size != 2048:
raise ValueError(
"Private Key must be 2048 bits"
)
return priv_key, priv_key.public_key()
# Now re-load it with OpenSSL's validation applied.
safe_priv_key = load()
return safe_priv_key, safe_priv_key.public_key()
def der_string_from_signing_key(private_key):
def der_string_from_signing_key(private_key: PrivateKey) -> bytes:
"""
Serializes a given RSA private key to a DER string
@ -98,14 +127,14 @@ def der_string_from_signing_key(private_key):
:returns: bytes representing `private_key`
"""
_validate_private_key(private_key)
return private_key.private_bytes(
return private_key.private_bytes( # type: ignore[attr-defined]
encoding=Encoding.DER,
format=PrivateFormat.PKCS8,
encryption_algorithm=NoEncryption(),
)
def der_string_from_verifying_key(public_key):
def der_string_from_verifying_key(public_key: PublicKey) -> bytes:
"""
Serializes a given RSA public key to a DER string.
@ -121,7 +150,7 @@ def der_string_from_verifying_key(public_key):
)
def create_verifying_key_from_string(public_key_der):
def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey:
"""
Create an RSA verifying key from a previously serialized public key
@ -134,15 +163,16 @@ def create_verifying_key_from_string(public_key_der):
public_key_der,
backend=default_backend(),
)
assert isinstance(pub_key, PublicKey)
return pub_key
def sign_data(private_key, data):
def sign_data(private_key: PrivateKey, data: bytes) -> bytes:
"""
:param private_key: the private part of a keypair returned from
`create_signing_keypair_from_string` or `create_signing_keypair`
:param bytes data: the bytes to sign
:param data: the bytes to sign
:returns: bytes which are a signature of the bytes given as `data`.
"""
@ -153,7 +183,7 @@ def sign_data(private_key, data):
hashes.SHA256(),
)
def verify_signature(public_key, alleged_signature, data):
def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None:
"""
:param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key`
@ -173,23 +203,23 @@ def verify_signature(public_key, alleged_signature, data):
raise BadSignature()
def _validate_public_key(public_key):
def _validate_public_key(public_key: PublicKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(public_key, rsa.RSAPublicKey):
raise ValueError(
"public_key must be an RSAPublicKey"
f"public_key must be an RSAPublicKey not {type(public_key)}"
)
def _validate_private_key(private_key):
def _validate_private_key(private_key: PrivateKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(private_key, rsa.RSAPrivateKey):
raise ValueError(
"private_key must be an RSAPrivateKey"
f"private_key must be an RSAPrivateKey not {type(private_key)}"
)

2
src/allmydata/history.py
View File

@ -20,7 +20,7 @@ class History(object):
MAX_UPLOAD_STATUSES = 10
MAX_MAPUPDATE_STATUSES = 20
MAX_PUBLISH_STATUSES = 20
MAX_RETRIEVE_STATUSES = 20
MAX_RETRIEVE_STATUSES = 40
def __init__(self, stats_provider=None):
self.stats_provider = stats_provider

2
src/allmydata/immutable/encode.py
View File

@ -262,6 +262,8 @@ class Encoder(object):
d.addCallback(lambda res: self.finish_hashing())
# These calls have to happen in order; layout.py now requires writes to
# be appended to the data written so far.
d.addCallback(lambda res:
self.send_crypttext_hash_tree_to_all_shareholders())
d.addCallback(lambda res: self.send_all_block_hash_trees())

138
src/allmydata/immutable/layout.py
View File

@ -1,21 +1,18 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import struct
from io import BytesIO
from attrs import define, field
from zope.interface import implementer
from twisted.internet import defer
from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \
FileTooLargeError, HASH_SIZE
from allmydata.util import mathutil, observer, pipeline, log
from allmydata.util import mathutil, observer, log
from allmydata.util.assertutil import precondition
from allmydata.storage.server import si_b2a
@ -107,19 +104,58 @@ def make_write_bucket_proxy(rref, server,
num_share_hashes, uri_extension_size)
return wbp
@define
class _WriteBuffer:
"""
Queue up small writes to be written in a single batched larger write.
"""
_batch_size: int
_to_write : BytesIO = field(factory=BytesIO)
_written_bytes : int = field(default=0)
def queue_write(self, data: bytes) -> bool:
"""
Queue a write. If the result is ``False``, no further action is needed
for now. If the result is some ``True``, it's time to call ``flush()``
and do a real write.
"""
self._to_write.write(data)
return self.get_queued_bytes() >= self._batch_size
def flush(self) -> tuple[int, bytes]:
"""Return offset and data to be written."""
offset = self._written_bytes
data = self._to_write.getvalue()
self._written_bytes += len(data)
self._to_write = BytesIO()
return (offset, data)
def get_queued_bytes(self) -> int:
"""Return number of queued, unwritten bytes."""
return self._to_write.tell()
def get_total_bytes(self) -> int:
"""Return how many bytes were written or queued in total."""
return self._written_bytes + self.get_queued_bytes()
@implementer(IStorageBucketWriter)
class WriteBucketProxy(object):
"""
Note: The various ``put_`` methods need to be called in the order in which the
bytes will get written.
"""
fieldsize = 4
fieldstruct = ">L"
def __init__(self, rref, server, data_size, block_size, num_segments,
num_share_hashes, uri_extension_size, pipeline_size=50000):
num_share_hashes, uri_extension_size, batch_size=1_000_000):
self._rref = rref
self._server = server
self._data_size = data_size
self._block_size = block_size
self._num_segments = num_segments
self._written_bytes = 0
effective_segments = mathutil.next_power_of_k(num_segments,2)
self._segment_hash_size = (2*effective_segments - 1) * HASH_SIZE
@ -130,11 +166,13 @@ class WriteBucketProxy(object):
self._create_offsets(block_size, data_size)
# k=3, max_segment_size=128KiB gives us a typical segment of 43691
# bytes. Setting the default pipeline_size to 50KB lets us get two
# segments onto the wire but not a third, which would keep the pipe
# filled.
self._pipeline = pipeline.Pipeline(pipeline_size)
# With a ~1MB batch size, max upload speed is 1MB/(round-trip latency)
# assuming the writing code waits for writes to finish, so 20MB/sec if
# latency is 50ms. In the US many people only have 1MB/sec upload speed
# as of 2022 (standard Comcast). For further discussion of how one
# might set batch sizes see
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3787#comment:1.
self._write_buffer = _WriteBuffer(batch_size)
def get_allocated_size(self):
return (self._offsets['uri_extension'] + self.fieldsize +
@ -179,7 +217,7 @@ class WriteBucketProxy(object):
return "<WriteBucketProxy for node %r>" % self._server.get_name()
def put_header(self):
return self._write(0, self._offset_data)
return self._queue_write(0, self._offset_data)
def put_block(self, segmentnum, data):
offset = self._offsets['data'] + segmentnum * self._block_size
@ -193,13 +231,13 @@ class WriteBucketProxy(object):
(self._block_size *
(self._num_segments - 1))),
len(data), self._block_size)
return self._write(offset, data)
return self._queue_write(offset, data)
def put_crypttext_hashes(self, hashes):
# plaintext_hash_tree precedes crypttext_hash_tree. It is not used, and
# so is not explicitly written, but we need to write everything, so
# fill it in with nulls.
d = self._write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size)
d = self._queue_write(self._offsets['plaintext_hash_tree'], b"\x00" * self._segment_hash_size)
d.addCallback(lambda _: self._really_put_crypttext_hashes(hashes))
return d
@ -212,7 +250,7 @@ class WriteBucketProxy(object):
precondition(offset + len(data) <= self._offsets['block_hashes'],
offset, len(data), offset+len(data),
self._offsets['block_hashes'])
return self._write(offset, data)
return self._queue_write(offset, data)
def put_block_hashes(self, blockhashes):
offset = self._offsets['block_hashes']
@ -223,7 +261,7 @@ class WriteBucketProxy(object):
precondition(offset + len(data) <= self._offsets['share_hashes'],
offset, len(data), offset+len(data),
self._offsets['share_hashes'])
return self._write(offset, data)
return self._queue_write(offset, data)
def put_share_hashes(self, sharehashes):
# sharehashes is a list of (index, hash) tuples, so they get stored
@ -237,29 +275,45 @@ class WriteBucketProxy(object):
precondition(offset + len(data) <= self._offsets['uri_extension'],
offset, len(data), offset+len(data),
self._offsets['uri_extension'])
return self._write(offset, data)
return self._queue_write(offset, data)
def put_uri_extension(self, data):
offset = self._offsets['uri_extension']
assert isinstance(data, bytes)
precondition(len(data) == self._uri_extension_size)
length = struct.pack(self.fieldstruct, len(data))
return self._write(offset, length+data)
return self._queue_write(offset, length+data)
def _write(self, offset, data):
# use a Pipeline to pipeline several writes together. TODO: another
# speedup would be to coalesce small writes into a single call: this
# would reduce the foolscap CPU overhead per share, but wouldn't
# reduce the number of round trips, so it might not be worth the
# effort.
self._written_bytes += len(data)
return self._pipeline.add(len(data),
self._rref.callRemote, "write", offset, data)
def _queue_write(self, offset, data):
"""
This queues up small writes to be written in a single batched larger
write.
Callers of this function are expected to queue the data in order, with
no holes. As such, the offset is technically unnecessary, but is used
to check the inputs. Possibly we should get rid of it.
"""
assert offset == self._write_buffer.get_total_bytes()
if self._write_buffer.queue_write(data):
return self._actually_write()
else:
return defer.succeed(False)
def _actually_write(self):
"""Write data to the server."""
offset, data = self._write_buffer.flush()
return self._rref.callRemote("write", offset, data)
def close(self):
assert self._written_bytes == self.get_allocated_size(), f"{self._written_bytes} != {self.get_allocated_size()}"
d = self._pipeline.add(0, self._rref.callRemote, "close")
d.addCallback(lambda ign: self._pipeline.flush())
assert self._write_buffer.get_total_bytes() == self.get_allocated_size(), (
f"{self._written_buffer.get_total_bytes_queued()} != {self.get_allocated_size()}"
)
if self._write_buffer.get_queued_bytes() > 0:
d = self._actually_write()
else:
# No data queued, don't send empty string write.
d = defer.succeed(True)
d.addCallback(lambda _: self._rref.callRemote("close"))
return d
def abort(self):
@ -371,16 +425,16 @@ class ReadBucketProxy(object):
self._fieldsize = fieldsize
self._fieldstruct = fieldstruct
for field in ( 'data',
'plaintext_hash_tree', # UNUSED
'crypttext_hash_tree',
'block_hashes',
'share_hashes',
'uri_extension',
):
for field_name in ( 'data',
'plaintext_hash_tree', # UNUSED
'crypttext_hash_tree',
'block_hashes',
'share_hashes',
'uri_extension',
):
offset = struct.unpack(fieldstruct, data[x:x+fieldsize])[0]
x += fieldsize
self._offsets[field] = offset
self._offsets[field_name] = offset
return self._offsets
def _get_block_data(self, unused, blocknum, blocksize, thisblocksize):

42
src/allmydata/mutable/common.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
MODE_CHECK = "MODE_CHECK" # query all peers
MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version
@ -17,6 +10,9 @@ MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial
MODE_READ = "MODE_READ"
MODE_REPAIR = "MODE_REPAIR" # query all peers, get the privkey
from allmydata.crypto import aes, rsa
from allmydata.util import hashutil
class NotWriteableError(Exception):
pass
@ -68,3 +64,33 @@ class CorruptShareError(BadShareError):
class UnknownVersionError(BadShareError):
"""The share we received was of a version we don't recognize."""
def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes:
"""
For SSK, encrypt a private ("signature") key using the writekey.
"""
encryptor = aes.create_encryptor(writekey)
crypttext = aes.encrypt_data(encryptor, privkey)
return crypttext
def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey:
"""
The inverse of ``encrypt_privkey``.
"""
decryptor = aes.create_decryptor(writekey)
privkey = aes.decrypt_data(decryptor, enc_privkey)
return privkey
def derive_mutable_keys(keypair: tuple[rsa.PublicKey, rsa.PrivateKey]) -> tuple[bytes, bytes, bytes]:
"""
Derive the SSK writekey, encrypted writekey, and fingerprint from the
public/private ("verification" / "signature") keypair.
"""
pubkey, privkey = keypair
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
privkey_s = rsa.der_string_from_signing_key(privkey)
writekey = hashutil.ssk_writekey_hash(privkey_s)
encprivkey = encrypt_privkey(writekey, privkey_s)
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
return writekey, encprivkey, fingerprint

42
src/allmydata/mutable/filenode.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import random
@ -16,8 +9,6 @@ from zope.interface import implementer
from twisted.internet import defer, reactor
from foolscap.api import eventually
from allmydata.crypto import aes
from allmydata.crypto import rsa
from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \
NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \
IMutableFileVersion, IWriteable
@ -28,8 +19,14 @@ from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \
from allmydata.monitor import Monitor
from allmydata.mutable.publish import Publish, MutableData,\
TransformingUploadable
from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \
UncoordinatedWriteError
from allmydata.mutable.common import (
MODE_READ,
MODE_WRITE,
MODE_CHECK,
UnrecoverableFileError,
UncoordinatedWriteError,
derive_mutable_keys,
)
from allmydata.mutable.servermap import ServerMap, ServermapUpdater
from allmydata.mutable.retrieve import Retrieve
from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer
@ -139,13 +136,10 @@ class MutableFileNode(object):
Deferred that fires (with the MutableFileNode instance you should
use) when it completes.
"""
(pubkey, privkey) = keypair
self._pubkey, self._privkey = pubkey, privkey
pubkey_s = rsa.der_string_from_verifying_key(self._pubkey)
privkey_s = rsa.der_string_from_signing_key(self._privkey)
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
self._pubkey, self._privkey = keypair
self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys(
keypair,
)
if version == MDMF_VERSION:
self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
@ -171,16 +165,6 @@ class MutableFileNode(object):
(contents, type(contents))
return contents(self)
def _encrypt_privkey(self, writekey, privkey):
encryptor = aes.create_encryptor(writekey)
crypttext = aes.encrypt_data(encryptor, privkey)
return crypttext
def _decrypt_privkey(self, enc_privkey):
decryptor = aes.create_decryptor(self._writekey)
privkey = aes.decrypt_data(decryptor, enc_privkey)
return privkey
def _populate_pubkey(self, pubkey):
self._pubkey = pubkey
def _populate_required_shares(self, required_shares):

17
src/allmydata/mutable/retrieve.py
View File

@ -1,15 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Don't import bytes and str, to prevent API leakage
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401
from __future__ import annotations
import time
@ -32,7 +24,7 @@ from allmydata import hashtree, codec
from allmydata.storage.server import si_b2a
from allmydata.mutable.common import CorruptShareError, BadShareError, \
UncoordinatedWriteError
UncoordinatedWriteError, decrypt_privkey
from allmydata.mutable.layout import MDMFSlotReadProxy
@implementer(IRetrieveStatus)
@ -931,9 +923,10 @@ class Retrieve(object):
def _try_to_validate_privkey(self, enc_privkey, reader, server):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
node_writekey = self._node.get_writekey()
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
if alleged_writekey != node_writekey:
self.log("invalid privkey from %s shnum %d" %
(reader, reader.shnum),
level=log.WEIRD, umid="YIw4tA")

27
src/allmydata/mutable/servermap.py
View File

@ -1,16 +1,8 @@
"""
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
# Doesn't import str to prevent API leakage on Python 2
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
from past.builtins import unicode
from six import ensure_str
import sys, time, copy
@ -29,7 +21,7 @@ from allmydata.storage.server import si_b2a
from allmydata.interfaces import IServermapUpdaterStatus
from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \
MODE_READ, MODE_REPAIR, CorruptShareError
MODE_READ, MODE_REPAIR, CorruptShareError, decrypt_privkey
from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy
@implementer(IServermapUpdaterStatus)
@ -203,8 +195,8 @@ class ServerMap(object):
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = verinfo
print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" %
(unicode(server.get_name(), "utf-8"), shnum,
seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N,
(str(server.get_name(), "utf-8"), shnum,
seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), k, N,
datalength), file=out)
if self._problems:
print("%d PROBLEMS" % len(self._problems), file=out)
@ -276,7 +268,7 @@ class ServerMap(object):
"""Take a versionid, return a string that describes it."""
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = verinfo
return "seq%d-%s" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"))
return "seq%d-%s" % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"))
def summarize_versions(self):
"""Return a string describing which versions we know about."""
@ -824,7 +816,7 @@ class ServermapUpdater(object):
def notify_server_corruption(self, server, shnum, reason):
if isinstance(reason, unicode):
if isinstance(reason, str):
reason = reason.encode("utf-8")
ss = server.get_storage_server()
ss.advise_corrupt_share(
@ -879,7 +871,7 @@ class ServermapUpdater(object):
# ok, it's a valid verinfo. Add it to the list of validated
# versions.
self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d"
% (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"),
% (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"),
ensure_str(server.get_name()), shnum,
k, n, segsize, datalen),
parent=lp)
@ -951,9 +943,10 @@ class ServermapUpdater(object):
writekey stored in my node. If it is valid, then I set the
privkey and encprivkey properties of the node.
"""
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
node_writekey = self._node.get_writekey()
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
if alleged_writekey != node_writekey:
self.log("invalid privkey from %r shnum %d" %
(server.get_name(), shnum),
parent=lp, level=log.WEIRD, umid="aJVccw")

20
src/allmydata/node.py
View File

@ -752,7 +752,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider):
def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
handler_overrides={}, **kwargs):
handler_overrides={}, force_foolscap=False, **kwargs):
"""
Create a Tub with the right options and handlers. It will be
ephemeral unless the caller provides certFile= in kwargs
@ -762,10 +762,16 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
:param dict tub_options: every key-value pair in here will be set in
the new Tub via `Tub.setOption`
:param bool force_foolscap: If True, only allow Foolscap, not just HTTPS
storage protocol.
"""
# We listen simulataneously for both Foolscap and HTTPS on the same port,
# We listen simultaneously for both Foolscap and HTTPS on the same port,
# so we have to create a special Foolscap Tub for that to work:
tub = create_tub_with_https_support(**kwargs)
if force_foolscap:
tub = Tub(**kwargs)
else:
tub = create_tub_with_https_support(**kwargs)
for (name, value) in list(tub_options.items()):
tub.setOption(name, value)
@ -956,14 +962,20 @@ def create_main_tub(config, tub_options,
# FIXME? "node.pem" was the CERTFILE option/thing
certfile = config.get_private_path("node.pem")
tub = create_tub(
tub_options,
default_connection_handlers,
foolscap_connection_handlers,
# TODO eventually we will want the default to be False, but for now we
# don't want to enable HTTP by default.
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3934
force_foolscap=config.get_config(
"storage", "force_foolscap", default=True, boolean=True
),
handler_overrides=handler_overrides,
certFile=certfile,
)
if portlocation is None:
log.msg("Tub is not listening")
else:

19
src/allmydata/nodemaker.py
View File

@ -1,17 +1,12 @@
"""
Ported to Python 3.
Create file nodes of various types.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import weakref
from zope.interface import implementer
from twisted.internet.defer import succeed
from allmydata.util.assertutil import precondition
from allmydata.interfaces import INodeMaker
from allmydata.immutable.literal import LiteralFileNode
@ -22,6 +17,7 @@ from allmydata.mutable.publish import MutableData
from allmydata.dirnode import DirectoryNode, pack_children
from allmydata.unknown import UnknownNode
from allmydata.blacklist import ProhibitedNode
from allmydata.crypto.rsa import PublicKey, PrivateKey
from allmydata import uri
@ -126,12 +122,15 @@ class NodeMaker(object):
return self._create_dirnode(filenode)
return None
def create_mutable_file(self, contents=None, version=None):
def create_mutable_file(self, contents=None, version=None, keypair: tuple[PublicKey, PrivateKey] | None = None):
if version is None:
version = self.mutable_file_default
n = MutableFileNode(self.storage_broker, self.secret_holder,
self.default_encoding_parameters, self.history)
d = self.key_generator.generate()
if keypair is None:
d = self.key_generator.generate()
else:
d = succeed(keypair)
d.addCallback(n.create_with_keys, contents, version=version)
d.addCallback(lambda res: n)
return d

16
src/allmydata/scripts/cli.py
View File

@ -180,10 +180,22 @@ class GetOptions(FileStoreOptions):
class PutOptions(FileStoreOptions):
optFlags = [
("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
]
]
optParameters = [
("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
]
("private-key-path", None, None,
"***Warning*** "
"It is possible to use this option to spoil the normal security properties of mutable objects. "
"It is also possible to corrupt or destroy data with this option. "
"Most users will not need this option and can ignore it. "
"For mutables only, "
"this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. "
"The private key must be handled at least as strictly as the resulting capability string. "
"A single private key must not be used for more than one mutable."
),
]
def parseArgs(self, arg1=None, arg2=None):
# see Examples below

42
src/allmydata/scripts/tahoe_put.py
View File

@ -1,23 +1,32 @@
"""
Ported to Python 3.
Implement the ``tahoe put`` command.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from io import BytesIO
from urllib.parse import quote as url_quote
from base64 import urlsafe_b64encode
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from twisted.python.filepath import FilePath
from allmydata.crypto.rsa import PrivateKey, der_string_from_signing_key
from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
UnknownAliasError
from allmydata.util.encodingutil import quote_output
def load_private_key(path: str) -> str:
"""
Load a private key from a file and return it in a format appropriate
to include in the HTTP request.
"""
privkey = load_pem_private_key(FilePath(path).getContent(), password=None)
assert isinstance(privkey, PrivateKey)
derbytes = der_string_from_signing_key(privkey)
return urlsafe_b64encode(derbytes).decode("ascii")
def put(options):
"""
@param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
@ -29,6 +38,10 @@ def put(options):
from_file = options.from_file
to_file = options.to_file
mutable = options['mutable']
if options["private-key-path"] is None:
private_key = None
else:
private_key = load_private_key(options["private-key-path"])
format = options['format']
if options['quiet']:
verbosity = 0
@ -79,6 +92,12 @@ def put(options):
queryargs = []
if mutable:
queryargs.append("mutable=true")
if private_key is not None:
queryargs.append(f"private-key={private_key}")
else:
if private_key is not None:
raise Exception("Can only supply a private key for mutables.")
if format:
queryargs.append("format=%s" % format)
if queryargs:
@ -92,10 +111,7 @@ def put(options):
if verbosity > 0:
print("waiting for file data on stdin..", file=stderr)
# We're uploading arbitrary files, so this had better be bytes:
if PY2:
stdinb = stdin
else:
stdinb = stdin.buffer
stdinb = stdin.buffer
data = stdinb.read()
infileobj = BytesIO(data)

54
src/allmydata/scripts/tahoe_run.py
View File

@ -21,7 +21,11 @@ from twisted.scripts import twistd
from twisted.python import usage
from twisted.python.filepath import FilePath
from twisted.python.reflect import namedAny
from twisted.internet.defer import maybeDeferred
from twisted.python.failure import Failure
from twisted.internet.defer import maybeDeferred, Deferred
from twisted.internet.protocol import Protocol
from twisted.internet.stdio import StandardIO
from twisted.internet.error import ReactorNotRunning
from twisted.application.service import Service
from allmydata.scripts.default_nodedir import _default_nodedir
@ -155,6 +159,8 @@ class DaemonizeTheRealService(Service, HookMixin):
def startService(self):
from twisted.internet import reactor
def start():
node_to_instance = {
u"client": lambda: maybeDeferred(namedAny("allmydata.client.create_client"), self.basedir),
@ -186,7 +192,7 @@ class DaemonizeTheRealService(Service, HookMixin):
)
)
else:
self.stderr.write("\nUnknown error\n")
self.stderr.write("\nUnknown error, here's the traceback:\n")
reason.printTraceback(self.stderr)
reactor.stop()
@ -194,12 +200,14 @@ class DaemonizeTheRealService(Service, HookMixin):
def created(srv):
srv.setServiceParent(self.parent)
# exiting on stdin-closed facilitates cleanup when run
# as a subprocess
on_stdin_close(reactor, reactor.stop)
d.addCallback(created)
d.addErrback(handle_config_error)
d.addBoth(self._call_hook, 'running')
return d
from twisted.internet import reactor
reactor.callWhenRunning(start)
@ -213,6 +221,46 @@ class DaemonizeTahoeNodePlugin(object):
return DaemonizeTheRealService(self.nodetype, self.basedir, so)
def on_stdin_close(reactor, fn):
"""
Arrange for the function `fn` to run when our stdin closes
"""
when_closed_d = Deferred()
class WhenClosed(Protocol):
"""
Notify a Deferred when our connection is lost .. as this is passed
to twisted's StandardIO class, it is used to detect our parent
going away.
"""
def connectionLost(self, reason):
when_closed_d.callback(None)
def on_close(arg):
try:
fn()
except ReactorNotRunning:
pass
except Exception:
# for our "exit" use-case failures will _mostly_ just be
# ReactorNotRunning (because we're already shutting down
# when our stdin closes) but no matter what "bad thing"
# happens we just want to ignore it .. although other
# errors might be interesting so we'll log those
print(Failure())
return arg
when_closed_d.addBoth(on_close)
# we don't need to do anything with this instance because it gets
# hooked into the reactor and thus remembered .. but we return it
# for Windows testing purposes.
return StandardIO(
proto=WhenClosed(),
reactor=reactor,
)
def run(reactor, config, runApp=twistd.runApp):
"""
Runs a Tahoe-LAFS node in the foreground.

160
src/allmydata/storage/http_client.py
View File

@ -20,7 +20,11 @@ from twisted.web.http_headers import Headers
from twisted.web import http
from twisted.web.iweb import IPolicyForHTTPS
from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed
from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
from twisted.internet.interfaces import (
IOpenSSLClientConnectionCreator,
IReactorTime,
IDelayedCall,
)
from twisted.internet.ssl import CertificateOptions
from twisted.web.client import Agent, HTTPConnectionPool
from zope.interface import implementer
@ -83,35 +87,35 @@ _SCHEMAS = {
"allocate_buckets": Schema(
"""
response = {
already-have: #6.258([* uint])
allocated: #6.258([* uint])
already-have: #6.258([0*256 uint])
allocated: #6.258([0*256 uint])
}
"""
),
"immutable_write_share_chunk": Schema(
"""
response = {
required: [* {begin: uint, end: uint}]
required: [0* {begin: uint, end: uint}]
}
"""
),
"list_shares": Schema(
"""
response = #6.258([* uint])
response = #6.258([0*256 uint])
"""
),
"mutable_read_test_write": Schema(
"""
response = {
"success": bool,
"data": {* share_number: [* bstr]}
"data": {0*256 share_number: [0* bstr]}
}
share_number = uint
"""
),
"mutable_list_shares": Schema(
"""
response = #6.258([* uint])
response = #6.258([0*256 uint])
"""
),
}
@ -124,16 +128,22 @@ class _LengthLimitedCollector:
"""
remaining_length: int
timeout_on_silence: IDelayedCall
f: BytesIO = field(factory=BytesIO)
def __call__(self, data: bytes):
self.timeout_on_silence.reset(60)
self.remaining_length -= len(data)
if self.remaining_length < 0:
raise ValueError("Response length was too long")
self.f.write(data)
def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[BinaryIO]:
def limited_content(
response,
clock: IReactorTime,
max_length: int = 30 * 1024 * 1024,
) -> Deferred[BinaryIO]:
"""
Like ``treq.content()``, but limit data read from the response to a set
length. If the response is longer than the max allowed length, the result
@ -142,39 +152,29 @@ def limited_content(response, max_length: int = 30 * 1024 * 1024) -> Deferred[Bi
A potentially useful future improvement would be using a temporary file to
store the content; since filesystem buffering means that would use memory
for small responses and disk for large responses.
This will time out if no data is received for 60 seconds; so long as a
trickle of data continues to arrive, it will continue to run.
"""
collector = _LengthLimitedCollector(max_length)
d = succeed(None)
timeout = clock.callLater(60, d.cancel)
collector = _LengthLimitedCollector(max_length, timeout)
# Make really sure everything gets called in Deferred context, treq might
# call collector directly...
d = succeed(None)
d.addCallback(lambda _: treq.collect(response, collector))
def done(_):
timeout.cancel()
collector.f.seek(0)
return collector.f
d.addCallback(done)
return d
def failed(f):
if timeout.active():
timeout.cancel()
return f
def _decode_cbor(response, schema: Schema):
"""Given HTTP response, return decoded CBOR body."""
def got_content(f: BinaryIO):
data = f.read()
schema.validate_cbor(data)
return loads(data)
if response.code > 199 and response.code < 300:
content_type = get_content_type(response.headers)
if content_type == CBOR_MIME_TYPE:
return limited_content(response).addCallback(got_content)
else:
raise ClientException(-1, "Server didn't send CBOR")
else:
return treq.content(response).addCallback(
lambda data: fail(ClientException(response.code, response.phrase, data))
)
return d.addCallbacks(done, failed)
@define
@ -276,42 +276,68 @@ class _StorageClientHTTPSPolicy:
)
@define
@define(hash=True)
class StorageClient(object):
"""
Low-level HTTP client that talks to the HTTP storage server.
"""
# If set, we're doing unit testing and we should call this with
# HTTPConnectionPool we create.
TEST_MODE_REGISTER_HTTP_POOL = None
@classmethod
def start_test_mode(cls, callback):
"""Switch to testing mode.
In testing mode we register the pool with test system using the given
callback so it can Do Things, most notably killing off idle HTTP
connections at test shutdown and, in some tests, in the midddle of the
test.
"""
cls.TEST_MODE_REGISTER_HTTP_POOL = callback
@classmethod
def stop_test_mode(cls):
"""Stop testing mode."""
cls.TEST_MODE_REGISTER_HTTP_POOL = None
# The URL is a HTTPS URL ("https://..."). To construct from a NURL, use
# ``StorageClient.from_nurl()``.
_base_url: DecodedURL
_swissnum: bytes
_treq: Union[treq, StubTreq, HTTPClient]
_clock: IReactorTime
@classmethod
def from_nurl(
cls, nurl: DecodedURL, reactor, persistent: bool = True
cls,
nurl: DecodedURL,
reactor,
) -> StorageClient:
"""
Create a ``StorageClient`` for the given NURL.
``persistent`` indicates whether to use persistent HTTP connections.
"""
assert nurl.fragment == "v=1"
assert nurl.scheme == "pb"
swissnum = nurl.path[0].encode("ascii")
certificate_hash = nurl.user.encode("ascii")
pool = HTTPConnectionPool(reactor)
pool.maxPersistentPerHost = 20
if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
treq_client = HTTPClient(
Agent(
reactor,
_StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
pool=HTTPConnectionPool(reactor, persistent=persistent),
pool=pool,
)
)
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
return cls(https_url, swissnum, treq_client)
return cls(https_url, swissnum, treq_client, reactor)
def relative_url(self, path):
"""Get a URL relative to the base URL."""
@ -337,6 +363,7 @@ class StorageClient(object):
write_enabler_secret=None,
headers=None,
message_to_serialize=None,
timeout: float = 60,
**kwargs,
):
"""
@ -345,6 +372,8 @@ class StorageClient(object):
If ``message_to_serialize`` is set, it will be serialized (by default
with CBOR) and set as the request body.
Default timeout is 60 seconds.
"""
headers = self._get_headers(headers)
@ -376,16 +405,37 @@ class StorageClient(object):
kwargs["data"] = dumps(message_to_serialize)
headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
return self._treq.request(method, url, headers=headers, **kwargs)
return self._treq.request(
method, url, headers=headers, timeout=timeout, **kwargs
)
def decode_cbor(self, response, schema: Schema):
"""Given HTTP response, return decoded CBOR body."""
def got_content(f: BinaryIO):
data = f.read()
schema.validate_cbor(data)
return loads(data)
if response.code > 199 and response.code < 300:
content_type = get_content_type(response.headers)
if content_type == CBOR_MIME_TYPE:
return limited_content(response, self._clock).addCallback(got_content)
else:
raise ClientException(-1, "Server didn't send CBOR")
else:
return treq.content(response).addCallback(
lambda data: fail(ClientException(response.code, response.phrase, data))
)
@define(hash=True)
class StorageClientGeneral(object):
"""
High-level HTTP APIs that aren't immutable- or mutable-specific.
"""
def __init__(self, client): # type: (StorageClient) -> None
self._client = client
_client: StorageClient
@inlineCallbacks
def get_version(self):
@ -394,7 +444,9 @@ class StorageClientGeneral(object):
"""
url = self._client.relative_url("/storage/v1/version")
response = yield self._client.request("GET", url)
decoded_response = yield _decode_cbor(response, _SCHEMAS["get_version"])
decoded_response = yield self._client.decode_cbor(
response, _SCHEMAS["get_version"]
)
returnValue(decoded_response)
@inlineCallbacks
@ -461,6 +513,9 @@ def read_share_chunk(
share_type, _encode_si(storage_index), share_number
)
)
# The default 60 second timeout is for getting the response, so it doesn't
# include the time it takes to download the body... so we will will deal
# with that later, via limited_content().
response = yield client.request(
"GET",
url,
@ -469,6 +524,7 @@ def read_share_chunk(
# but Range constructor does that the conversion for us.
{"range": [Range("bytes", [(offset, offset + length)]).to_header()]}
),
unbuffered=True, # Don't buffer the response in memory.
)
if response.code == http.NO_CONTENT:
@ -491,7 +547,7 @@ def read_share_chunk(
raise ValueError("Server sent more than we asked for?!")
# It might also send less than we asked for. That's (probably) OK, e.g.
# if we went past the end of the file.
body = yield limited_content(response, supposed_length)
body = yield limited_content(response, client._clock, supposed_length)
body.seek(0, SEEK_END)
actual_length = body.tell()
if actual_length != supposed_length:
@ -534,7 +590,7 @@ async def advise_corrupt_share(
)
@define
@define(hash=True)
class StorageClientImmutables(object):
"""
APIs for interacting with immutables.
@ -578,7 +634,9 @@ class StorageClientImmutables(object):
upload_secret=upload_secret,
message_to_serialize=message,
)
decoded_response = yield _decode_cbor(response, _SCHEMAS["allocate_buckets"])
decoded_response = yield self._client.decode_cbor(
response, _SCHEMAS["allocate_buckets"]
)
returnValue(
ImmutableCreateResult(
already_have=decoded_response["already-have"],
@ -654,7 +712,9 @@ class StorageClientImmutables(object):
raise ClientException(
response.code,
)
body = yield _decode_cbor(response, _SCHEMAS["immutable_write_share_chunk"])
body = yield self._client.decode_cbor(
response, _SCHEMAS["immutable_write_share_chunk"]
)
remaining = RangeMap()
for chunk in body["required"]:
remaining.set(True, chunk["begin"], chunk["end"])
@ -683,7 +743,7 @@ class StorageClientImmutables(object):
url,
)
if response.code == http.OK:
body = yield _decode_cbor(response, _SCHEMAS["list_shares"])
body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
returnValue(set(body))
else:
raise ClientException(response.code)
@ -800,7 +860,9 @@ class StorageClientMutables:
message_to_serialize=message,
)
if response.code == http.OK:
result = await _decode_cbor(response, _SCHEMAS["mutable_read_test_write"])
result = await self._client.decode_cbor(
response, _SCHEMAS["mutable_read_test_write"]
)
return ReadTestWriteResult(success=result["success"], reads=result["data"])
else:
raise ClientException(response.code, (await response.content()))
@ -829,7 +891,9 @@ class StorageClientMutables:
)
response = await self._client.request("GET", url)
if response.code == http.OK:
return await _decode_cbor(response, _SCHEMAS["mutable_list_shares"])
return await self._client.decode_cbor(
response, _SCHEMAS["mutable_list_shares"]
)
else:
raise ClientException(response.code)

72
src/allmydata/storage/http_server.py
View File

@ -9,6 +9,8 @@ from functools import wraps
from base64 import b64decode
import binascii
from tempfile import TemporaryFile
from os import SEEK_END, SEEK_SET
import mmap
from cryptography.x509 import Certificate as CryptoCertificate
from zope.interface import implementer
@ -39,7 +41,7 @@ from cryptography.x509 import load_pem_x509_certificate
# TODO Make sure to use pure Python versions?
from cbor2 import dump, loads
import cbor2
from pycddl import Schema, ValidationError as CDDLValidationError
from .server import StorageServer
from .http_common import (
@ -100,7 +102,7 @@ def _authorization_decorator(required_secrets):
@wraps(f)
def route(self, request, *args, **kwargs):
if not timing_safe_compare(
request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode(
request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
"utf-8"
),
swissnum_auth_header(self._swissnum),
@ -260,7 +262,7 @@ _SCHEMAS = {
"allocate_buckets": Schema(
"""
request = {
share-numbers: #6.258([*256 uint])
share-numbers: #6.258([0*256 uint])
allocated-size: uint
}
"""
@ -276,15 +278,13 @@ _SCHEMAS = {
"""
request = {
"test-write-vectors": {
; TODO Add length limit here, after
; https://github.com/anweiss/cddl/issues/128 is fixed
* share_number => {
"test": [*30 {"offset": uint, "size": uint, "specimen": bstr}]
"write": [*30 {"offset": uint, "data": bstr}]
0*256 share_number : {
"test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}]
"write": [* {"offset": uint, "data": bstr}]
"new-length": uint / null
}
}
"read-vector": [*30 {"offset": uint, "size": uint}]
"read-vector": [0*30 {"offset": uint, "size": uint}]
}
share_number = uint
"""
@ -517,7 +517,7 @@ class HTTPServer(object):
if accept.best == CBOR_MIME_TYPE:
request.setHeader("Content-Type", CBOR_MIME_TYPE)
f = TemporaryFile()
dump(data, f)
cbor2.dump(data, f)
def read_data(offset: int, length: int) -> bytes:
f.seek(offset)
@ -529,27 +529,47 @@ class HTTPServer(object):
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861
raise _HTTPError(http.NOT_ACCEPTABLE)
def _read_encoded(self, request, schema: Schema) -> Any:
def _read_encoded(
self, request, schema: Schema, max_size: int = 1024 * 1024
) -> Any:
"""
Read encoded request body data, decoding it with CBOR by default.
Somewhat arbitrarily, limit body size to 1MB; this may be too low, we
may want to customize per query type, but this is the starting point
for now.
Somewhat arbitrarily, limit body size to 1MiB by default.
"""
content_type = get_content_type(request.requestHeaders)
if content_type == CBOR_MIME_TYPE:
# Read 1 byte more than 1MB. We expect length to be 1MB or
# less; if it's more assume it's not a legitimate message.
message = request.content.read(1024 * 1024 + 1)
if len(message) > 1024 * 1024:
raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE)
schema.validate_cbor(message)
result = loads(message)
return result
else:
if content_type != CBOR_MIME_TYPE:
raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE)
# Make sure it's not too large:
request.content.seek(SEEK_END, 0)
if request.content.tell() > max_size:
raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE)
request.content.seek(SEEK_SET, 0)
# We don't want to load the whole message into memory, cause it might
# be quite large. The CDDL validator takes a read-only bytes-like
# thing. Luckily, for large request bodies twisted.web will buffer the
# data in a file, so we can use mmap() to get a memory view. The CDDL
# validator will not make a copy, so it won't increase memory usage
# beyond that.
try:
fd = request.content.fileno()
except (ValueError, OSError):
fd = -1
if fd >= 0:
# It's a file, so we can use mmap() to save memory.
message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
else:
message = request.content.read()
schema.validate_cbor(message)
# The CBOR parser will allocate more memory, but at least we can feed
# it the file-like object, so that if it's large it won't be make two
# copies.
request.content.seek(SEEK_SET, 0)
return cbor2.load(request.content)
##### Generic APIs #####
@_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
@ -748,7 +768,9 @@ class HTTPServer(object):
)
def mutable_read_test_write(self, request, authorization, storage_index):
"""Read/test/write combined operation for mutables."""
rtw_request = self._read_encoded(request, _SCHEMAS["mutable_read_test_write"])
rtw_request = self._read_encoded(
request, _SCHEMAS["mutable_read_test_write"], max_size=2**48
)
secrets = (
authorization[Secrets.WRITE_ENABLER],
authorization[Secrets.LEASE_RENEW],

295
src/allmydata/storage_client.py
View File

@ -30,6 +30,8 @@ Ported to Python 3.
#
# 6: implement other sorts of IStorageClient classes: S3, etc
from __future__ import annotations
from six import ensure_text
from typing import Union
@ -41,13 +43,16 @@ import hashlib
from configparser import NoSectionError
import attr
from hyperlink import DecodedURL
from zope.interface import (
Attribute,
Interface,
implementer,
)
from twisted.python.failure import Failure
from twisted.web import http
from twisted.internet import defer
from twisted.internet.task import LoopingCall
from twisted.internet import defer, reactor
from twisted.application import service
from twisted.plugin import (
getPlugins,
@ -85,6 +90,8 @@ from allmydata.storage.http_client import (
ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException
)
ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs"
# who is responsible for de-duplication?
# both?
@ -109,8 +116,8 @@ class StorageClientConfig(object):
:ivar preferred_peers: An iterable of the server-ids (``bytes``) of the
storage servers where share placement is preferred, in order of
decreasing preference. See the *[client]peers.preferred*
documentation for details.
decreasing preference. See the *[client]peers.preferred* documentation
for details.
:ivar dict[unicode, dict[unicode, unicode]] storage_plugins: A mapping from
names of ``IFoolscapStoragePlugin`` configured in *tahoe.cfg* to the
@ -286,6 +293,10 @@ class StorageFarmBroker(service.MultiService):
by the given announcement.
"""
assert isinstance(server_id, bytes)
if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0:
s = HTTPNativeStorageServer(server_id, server["ann"])
s.on_status_changed(lambda _: self._got_connection())
return s
handler_overrides = server.get("connections", {})
gm_verifier = create_grid_manager_verifier(
self.storage_client_config.grid_manager_keys,
@ -569,6 +580,45 @@ class IFoolscapStorageServer(Interface):
"""
def _parse_announcement(server_id: bytes, furl: bytes, ann: dict) -> tuple[str, bytes, bytes, bytes, bytes]:
"""
Parse the furl and announcement, return:
(nickname, permutation_seed, tubid, short_description, long_description)
"""
m = re.match(br'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
tubid = base32.a2b(tubid_s)
if "permutation-seed-base32" in ann:
seed = ann["permutation-seed-base32"]
if isinstance(seed, str):
seed = seed.encode("utf-8")
ps = base32.a2b(seed)
elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id):
ps = base32.a2b(server_id[3:])
else:
log.msg("unable to parse serverid '%(server_id)s as pubkey, "
"hashing it to get permutation-seed, "
"may not converge with other clients",
server_id=server_id,
facility="tahoe.storage_broker",
level=log.UNUSUAL, umid="qu86tw")
ps = hashlib.sha256(server_id).digest()
permutation_seed = ps
assert server_id
long_description = server_id
if server_id.startswith(b"v0-"):
# remove v0- prefix from abbreviated name
short_description = server_id[3:3+8]
else:
short_description = server_id[:8]
nickname = ann.get("nickname", "")
return (nickname, permutation_seed, tubid, short_description, long_description)
@implementer(IFoolscapStorageServer)
@attr.s(frozen=True)
class _FoolscapStorage(object):
@ -613,43 +663,13 @@ class _FoolscapStorage(object):
The furl will be a Unicode string on Python 3; on Python 2 it will be
either a native (bytes) string or a Unicode string.
"""
furl = furl.encode("utf-8")
m = re.match(br'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
tubid = base32.a2b(tubid_s)
if "permutation-seed-base32" in ann:
seed = ann["permutation-seed-base32"]
if isinstance(seed, str):
seed = seed.encode("utf-8")
ps = base32.a2b(seed)
elif re.search(br'^v0-[0-9a-zA-Z]{52}$', server_id):
ps = base32.a2b(server_id[3:])
else:
log.msg("unable to parse serverid '%(server_id)s as pubkey, "
"hashing it to get permutation-seed, "
"may not converge with other clients",
server_id=server_id,
facility="tahoe.storage_broker",
level=log.UNUSUAL, umid="qu86tw")
ps = hashlib.sha256(server_id).digest()
permutation_seed = ps
assert server_id
long_description = server_id
if server_id.startswith(b"v0-"):
# remove v0- prefix from abbreviated name
short_description = server_id[3:3+8]
else:
short_description = server_id[:8]
nickname = ann.get("nickname", "")
(nickname, permutation_seed, tubid, short_description, long_description) = _parse_announcement(server_id, furl.encode("utf-8"), ann)
return cls(
nickname=nickname,
permutation_seed=permutation_seed,
tubid=tubid,
storage_server=storage_server,
furl=furl,
furl=furl.encode("utf-8"),
short_description=short_description,
long_description=long_description,
)
@ -731,6 +751,16 @@ def _storage_from_foolscap_plugin(node_config, config, announcement, get_rref):
raise AnnouncementNotMatched()
def _available_space_from_version(version):
if version is None:
return None
protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict())
available_space = protocol_v1_version.get(b'available-space')
if available_space is None:
available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None)
return available_space
@implementer(IServer)
class NativeStorageServer(service.MultiService):
"""I hold information about a storage server that we want to connect to.
@ -910,13 +940,7 @@ class NativeStorageServer(service.MultiService):
def get_available_space(self):
version = self.get_version()
if version is None:
return None
protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict())
available_space = protocol_v1_version.get(b'available-space')
if available_space is None:
available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None)
return available_space
return _available_space_from_version(version)
def start_connecting(self, trigger_cb):
self._tub = self._tub_maker(self._handler_overrides)
@ -978,6 +1002,164 @@ class NativeStorageServer(service.MultiService):
# used when the broker wants us to hurry up
self._reconnector.reset()
@implementer(IServer)
class HTTPNativeStorageServer(service.MultiService):
"""
Like ``NativeStorageServer``, but for HTTP clients.
The notion of being "connected" is less meaningful for HTTP; we just poll
occasionally, and if we've succeeded at last poll, we assume we're
"connected".
"""
def __init__(self, server_id: bytes, announcement, reactor=reactor):
service.MultiService.__init__(self)
assert isinstance(server_id, bytes)
self._server_id = server_id
self.announcement = announcement
self._on_status_changed = ObserverList()
self._reactor = reactor
furl = announcement["anonymous-storage-FURL"].encode("utf-8")
(
self._nickname,
self._permutation_seed,
self._tubid,
self._short_description,
self._long_description
) = _parse_announcement(server_id, furl, announcement)
# TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs?
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935
nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0])
self._istorage_server = _HTTPStorageServer.from_http_client(
StorageClient.from_nurl(nurl, reactor)
)
self._connection_status = connection_status.ConnectionStatus.unstarted()
self._version = None
self._last_connect_time = None
self._connecting_deferred = None
def get_permutation_seed(self):
return self._permutation_seed
def get_name(self):
return self._short_description
def get_longname(self):
return self._long_description
def get_tubid(self):
return self._tubid
def get_lease_seed(self):
# Apparently this is what Foolscap version above does?!
return self._tubid
def get_foolscap_write_enabler_seed(self):
return self._tubid
def get_nickname(self):
return self._nickname
def on_status_changed(self, status_changed):
"""
:param status_changed: a callable taking a single arg (the
NativeStorageServer) that is notified when we become connected
"""
return self._on_status_changed.subscribe(status_changed)
# Special methods used by copy.copy() and copy.deepcopy(). When those are
# used in allmydata.immutable.filenode to copy CheckResults during
# repair, we want it to treat the IServer instances as singletons, and
# not attempt to duplicate them..
def __copy__(self):
return self
def __deepcopy__(self, memodict):
return self
def __repr__(self):
return "<HTTPNativeStorageServer for %r>" % self.get_name()
def get_serverid(self):
return self._server_id
def get_version(self):
return self._version
def get_announcement(self):
return self.announcement
def get_connection_status(self):
return self._connection_status
def is_connected(self):
return self._connection_status.connected
def get_available_space(self):
version = self.get_version()
return _available_space_from_version(version)
def start_connecting(self, trigger_cb):
self._lc = LoopingCall(self._connect)
self._lc.start(1, True)
def _got_version(self, version):
self._last_connect_time = time.time()
self._version = version
self._connection_status = connection_status.ConnectionStatus(
True, "connected", [], self._last_connect_time, self._last_connect_time
)
self._on_status_changed.notify(self)
def _failed_to_connect(self, reason):
self._connection_status = connection_status.ConnectionStatus(
False, f"failure: {reason}", [], self._last_connect_time, self._last_connect_time
)
self._on_status_changed.notify(self)
def get_storage_server(self):
"""
See ``IServer.get_storage_server``.
"""
if self._connection_status.summary == "unstarted":
return None
return self._istorage_server
def stop_connecting(self):
self._lc.stop()
if self._connecting_deferred is not None:
self._connecting_deferred.cancel()
def try_to_connect(self):
self._connect()
def _connect(self):
result = self._istorage_server.get_version()
def remove_connecting_deferred(result):
self._connecting_deferred = None
return result
# Set a short timeout since we're relying on this for server liveness.
self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth(
remove_connecting_deferred).addCallbacks(
self._got_version,
self._failed_to_connect
)
def stopService(self):
if self._connecting_deferred is not None:
self._connecting_deferred.cancel()
result = service.MultiService.stopService(self)
if self._lc.running:
self._lc.stop()
self._failed_to_connect("shut down")
return result
class UnknownServerTypeError(Exception):
pass
@ -1094,7 +1276,7 @@ class _StorageServer(object):
@attr.s
@attr.s(hash=True)
class _FakeRemoteReference(object):
"""
Emulate a Foolscap RemoteReference, calling a local object instead.
@ -1119,7 +1301,7 @@ class _HTTPBucketWriter(object):
storage_index = attr.ib(type=bytes)
share_number = attr.ib(type=int)
upload_secret = attr.ib(type=bytes)
finished = attr.ib(type=bool, default=False)
finished = attr.ib(type=defer.Deferred[bool], factory=defer.Deferred)
def abort(self):
return self.client.abort_upload(self.storage_index, self.share_number,
@ -1131,18 +1313,27 @@ class _HTTPBucketWriter(object):
self.storage_index, self.share_number, self.upload_secret, offset, data
)
if result.finished:
self.finished = True
self.finished.callback(True)
defer.returnValue(None)
def close(self):
# A no-op in HTTP protocol.
if not self.finished:
return defer.fail(RuntimeError("You didn't finish writing?!"))
return defer.succeed(None)
# We're not _really_ closed until all writes have succeeded and we
# finished writing all the data.
return self.finished
def _ignore_404(failure: Failure) -> Union[Failure, None]:
"""
Useful for advise_corrupt_share(), since it swallows unknown share numbers
in Foolscap.
"""
if failure.check(HTTPClientException) and failure.value.code == http.NOT_FOUND:
return None
else:
return failure
@attr.s
@attr.s(hash=True)
class _HTTPBucketReader(object):
"""
Emulate a ``RIBucketReader``, but use HTTP protocol underneath.
@ -1160,7 +1351,7 @@ class _HTTPBucketReader(object):
return self.client.advise_corrupt_share(
self.storage_index, self.share_number,
str(reason, "utf-8", errors="backslashreplace")
)
).addErrback(_ignore_404)
# WORK IN PROGRESS, for now it doesn't actually implement whole thing.
@ -1260,7 +1451,7 @@ class _HTTPStorageServer(object):
raise ValueError("Unknown share type")
return client.advise_corrupt_share(
storage_index, shnum, str(reason, "utf-8", errors="backslashreplace")
)
).addErrback(_ignore_404)
@defer.inlineCallbacks
def slot_readv(self, storage_index, shares, readv):

210
src/allmydata/test/_win_subprocess.py
View File

@ -1,210 +0,0 @@
"""
This module is only necessary on Python 2. Once Python 2 code is dropped, it
can be deleted.
"""
from future.utils import PY3
if PY3:
raise RuntimeError("Just use subprocess.Popen")
# This is necessary to pacify flake8 on Python 3, while we're still supporting
# Python 2.
from past.builtins import unicode
# -*- coding: utf-8 -*-
## Copyright (C) 2021 Valentin Lab
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted provided that the following conditions
## are met:
##
## 1. Redistributions of source code must retain the above copyright
## notice, this list of conditions and the following disclaimer.
##
## 2. Redistributions in binary form must reproduce the above
## copyright notice, this list of conditions and the following
## disclaimer in the documentation and/or other materials provided
## with the distribution.
##
## 3. Neither the name of the copyright holder nor the names of its
## contributors may be used to endorse or promote products derived
## from this software without specific prior written permission.
##
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
## OF THE POSSIBILITY OF SUCH DAMAGE.
##
## issue: https://bugs.python.org/issue19264
# See allmydata/windows/fixups.py
import sys
assert sys.platform == "win32"
import os
import ctypes
import subprocess
import _subprocess
from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \
Structure, sizeof, c_wchar, WinError
from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \
HANDLE
##
## Types
##
CREATE_UNICODE_ENVIRONMENT = 0x00000400
LPCTSTR = c_char_p
LPTSTR = c_wchar_p
LPSECURITY_ATTRIBUTES = c_void_p
LPBYTE = ctypes.POINTER(BYTE)
class STARTUPINFOW(Structure):
_fields_ = [
("cb", DWORD), ("lpReserved", LPWSTR),
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
("dwX", DWORD), ("dwY", DWORD),
("dwXSize", DWORD), ("dwYSize", DWORD),
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
("wShowWindow", WORD), ("cbReserved2", WORD),
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
("hStdOutput", HANDLE), ("hStdError", HANDLE),
]
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
class PROCESS_INFORMATION(Structure):
_fields_ = [
("hProcess", HANDLE), ("hThread", HANDLE),
("dwProcessId", DWORD), ("dwThreadId", DWORD),
]
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
class DUMMY_HANDLE(ctypes.c_void_p):
def __init__(self, *a, **kw):
super(DUMMY_HANDLE, self).__init__(*a, **kw)
self.closed = False
def Close(self):
if not self.closed:
windll.kernel32.CloseHandle(self)
self.closed = True
def __int__(self):
return self.value
CreateProcessW = windll.kernel32.CreateProcessW
CreateProcessW.argtypes = [
LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES,
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR,
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
]
CreateProcessW.restype = BOOL
##
## Patched functions/classes
##
def CreateProcess(executable, args, _p_attr, _t_attr,
inherit_handles, creation_flags, env, cwd,
startup_info):
"""Create a process supporting unicode executable and args for win32
Python implementation of CreateProcess using CreateProcessW for Win32
"""
si = STARTUPINFOW(
dwFlags=startup_info.dwFlags,
wShowWindow=startup_info.wShowWindow,
cb=sizeof(STARTUPINFOW),
## XXXvlab: not sure of the casting here to ints.
hStdInput=int(startup_info.hStdInput),
hStdOutput=int(startup_info.hStdOutput),
hStdError=int(startup_info.hStdError),
)
wenv = None
if env is not None:
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
env = (unicode("").join([
unicode("%s=%s\0") % (k, v)
for k, v in env.items()])) + unicode("\0")
wenv = (c_wchar * len(env))()
wenv.value = env
pi = PROCESS_INFORMATION()
creation_flags |= CREATE_UNICODE_ENVIRONMENT
if CreateProcessW(executable, args, None, None,
inherit_handles, creation_flags,
wenv, cwd, byref(si), byref(pi)):
return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
pi.dwProcessId, pi.dwThreadId)
raise WinError()
class Popen(subprocess.Popen):
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell, to_close,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
There are only 2 little changes concerning the construction of
the the final string in shell mode: we preempt the creation of
the command string when shell is True, because original function
will try to encode unicode args which we want to avoid to be able to
sending it as-is to ``CreateProcess``.
"""
if not isinstance(args, subprocess.types.StringTypes):
args = subprocess.list2cmdline(args)
if startupinfo is None:
startupinfo = subprocess.STARTUPINFO()
if shell:
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = _subprocess.SW_HIDE
comspec = os.environ.get("COMSPEC", unicode("cmd.exe"))
args = unicode('{} /c "{}"').format(comspec, args)
if (_subprocess.GetVersion() >= 0x80000000 or
os.path.basename(comspec).lower() == "command.com"):
w9xpopen = self._find_w9xpopen()
args = unicode('"%s" %s') % (w9xpopen, args)
creationflags |= _subprocess.CREATE_NEW_CONSOLE
cp = _subprocess.CreateProcess
_subprocess.CreateProcess = CreateProcess
try:
super(Popen, self)._execute_child(
args, executable,
preexec_fn, close_fds, cwd, env, universal_newlines,
startupinfo, creationflags, False, to_close, p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite,
)
finally:
_subprocess.CreateProcess = cp

79
src/allmydata/test/cli/test_put.py
View File

@ -1,19 +1,18 @@
"""
Ported to Python 3.
Tests for the ``tahoe put`` CLI tool.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing import Callable, Awaitable, TypeVar, Any
import os.path
from twisted.trial import unittest
from twisted.python import usage
from twisted.python.filepath import FilePath
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from allmydata.crypto.rsa import PrivateKey
from allmydata.uri import from_string
from allmydata.util import fileutil
from allmydata.scripts.common import get_aliases
from allmydata.scripts import cli
@ -22,6 +21,9 @@ from ..common_util import skip_if_cannot_represent_filename
from allmydata.util.encodingutil import get_io_encoding
from allmydata.util.fileutil import abspath_expanduser_unicode
from .common import CLITestMixin
from allmydata.mutable.common import derive_mutable_keys
T = TypeVar("T")
class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
@ -215,6 +217,65 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
return d
async def test_unlinked_mutable_specified_private_key(self) -> None:
"""
A new unlinked mutable can be created using a specified private
key.
"""
self.basedir = "cli/Put/unlinked-mutable-with-key"
await self._test_mutable_specified_key(
lambda do_cli, pempath, datapath: do_cli(
"put", "--mutable", "--private-key-path", pempath.path,
stdin=datapath.getContent(),
),
)
async def test_linked_mutable_specified_private_key(self) -> None:
"""
A new linked mutable can be created using a specified private key.
"""
self.basedir = "cli/Put/linked-mutable-with-key"
await self._test_mutable_specified_key(
lambda do_cli, pempath, datapath: do_cli(
"put", "--mutable", "--private-key-path", pempath.path, datapath.path,
),
)
async def _test_mutable_specified_key(
self,
run: Callable[[Any, FilePath, FilePath], Awaitable[tuple[int, bytes, bytes]]],
) -> None:
"""
A helper for testing mutable creation.
:param run: A function to do the creation. It is called with
``self.do_cli`` and the path to a private key PEM file and a data
file. It returns whatever ``do_cli`` returns.
"""
self.set_up_grid(oneshare=True)
pempath = FilePath(__file__).parent().sibling("data").child("openssl-rsa-2048.txt")
datapath = FilePath(self.basedir).child("data")
datapath.setContent(b"Hello world" * 1024)
(rc, out, err) = await run(self.do_cli, pempath, datapath)
self.assertEqual(rc, 0, (out, err))
cap = from_string(out.strip())
# The capability is derived from the key we specified.
privkey = load_pem_private_key(pempath.getContent(), password=None)
assert isinstance(privkey, PrivateKey)
pubkey = privkey.public_key()
writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
self.assertEqual(
(writekey, fingerprint),
(cap.writekey, cap.fingerprint),
)
# Also the capability we were given actually refers to the data we
# uploaded.
(rc, out, err) = await self.do_cli("get", out.strip())
self.assertEqual(rc, 0, (out, err))
self.assertEqual(out, datapath.getContent().decode("ascii"))
def test_mutable(self):
# echo DATA1 | tahoe put --mutable - uploaded.txt
# echo DATA2 | tahoe put - uploaded.txt # should modify-in-place

78
src/allmydata/test/common.py
View File

@ -1,14 +1,8 @@
"""
Ported to Python 3.
Functionality related to a lot of the test suite.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2, native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import chr as byteschr
__all__ = [
@ -111,25 +105,15 @@ from allmydata.scripts.common import (
from ..crypto import (
ed25519,
rsa,
)
from .eliotutil import (
EliotLoggedRunTest,
)
from .common_util import ShouldFailMixin # noqa: F401
if sys.platform == "win32" and PY2:
# Python 2.7 doesn't have good options for launching a process with
# non-ASCII in its command line. So use this alternative that does a
# better job. However, only use it on Windows because it doesn't work
# anywhere else.
from ._win_subprocess import (
Popen,
)
else:
from subprocess import (
Popen,
)
from subprocess import (
Popen,
PIPE,
)
@ -298,7 +282,7 @@ class UseNode(object):
plugin_config = attr.ib()
storage_plugin = attr.ib()
basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str),
introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
converter=six.ensure_str)
node_config = attr.ib(default=attr.Factory(dict))
@ -639,15 +623,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
MUTABLE_SIZELIMIT = 10000
def __init__(self, storage_broker, secret_holder,
default_encoding_parameters, history, all_contents):
_public_key: rsa.PublicKey | None
_private_key: rsa.PrivateKey | None
def __init__(self,
storage_broker,
secret_holder,
default_encoding_parameters,
history,
all_contents,
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None
):
self.all_contents = all_contents
self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
self.init_from_cap(make_mutable_file_cap())
self.file_types: dict[bytes, int] = {} # storage index => MDMF_VERSION or SDMF_VERSION
self.init_from_cap(make_mutable_file_cap(keypair))
self._k = default_encoding_parameters['k']
self._segsize = default_encoding_parameters['max_segment_size']
def create(self, contents, key_generator=None, keysize=None,
version=SDMF_VERSION):
if keypair is None:
self._public_key = self._private_key = None
else:
self._public_key, self._private_key = keypair
def create(self, contents, version=SDMF_VERSION):
if version == MDMF_VERSION and \
isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
uri.WriteableSSKFileURI)):
@ -843,9 +840,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
return defer.succeed(consumer)
def make_mutable_file_cap():
return uri.WriteableSSKFileURI(writekey=os.urandom(16),
fingerprint=os.urandom(32))
def make_mutable_file_cap(
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
) -> uri.WriteableSSKFileURI:
"""
Create a local representation of a mutable object.
:param keypair: If None, a random keypair will be generated for the new
object. Otherwise, this is the keypair for that object.
"""
if keypair is None:
writekey = os.urandom(16)
fingerprint = os.urandom(32)
else:
pubkey, privkey = keypair
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
privkey_s = rsa.der_string_from_signing_key(privkey)
writekey = hashutil.ssk_writekey_hash(privkey_s)
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
return uri.WriteableSSKFileURI(
writekey=writekey, fingerprint=fingerprint,
)
def make_mdmf_mutable_file_cap():
return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
@ -875,7 +891,7 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
encoding_params['max_segment_size'] = 128*1024
filenode = FakeMutableFileNode(None, None, encoding_params, None,
all_contents)
all_contents, None)
filenode.init_from_cap(cap)
if mdmf:
filenode.create(MutableData(contents), version=MDMF_VERSION)

109
src/allmydata/test/common_system.py
View File

@ -5,22 +5,14 @@ in ``allmydata.test.test_system``.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Don't import bytes since it causes issues on (so far unported) modules on Python 2.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401
from typing import Optional
import os
from functools import partial
from twisted.internet import reactor
from twisted.internet import defer
from twisted.internet.defer import inlineCallbacks
from twisted.internet.task import deferLater
from twisted.application import service
from foolscap.api import flushEventualQueue
@ -28,6 +20,12 @@ from foolscap.api import flushEventualQueue
from allmydata import client
from allmydata.introducer.server import create_introducer
from allmydata.util import fileutil, log, pollmixin
from allmydata.util.deferredutil import async_to_deferred
from allmydata.storage import http_client
from allmydata.storage_client import (
NativeStorageServer,
HTTPNativeStorageServer,
)
from twisted.python.filepath import (
FilePath,
@ -642,9 +640,51 @@ def _render_section_values(values):
))
@async_to_deferred
async def spin_until_cleanup_done(value=None, timeout=10):
"""
At the end of the test, spin until the reactor has no more DelayedCalls
and file descriptors (or equivalents) registered. This prevents dirty
reactor errors, while also not hard-coding a fixed amount of time, so it
can finish faster on faster computers.
There is also a timeout: if it takes more than 10 seconds (by default) for
the remaining reactor state to clean itself up, the presumption is that it
will never get cleaned up and the spinning stops.
Make sure to run as last thing in tearDown.
"""
def num_fds():
if hasattr(reactor, "handles"):
# IOCP!
return len(reactor.handles)
else:
# Normal reactor; having internal readers still registered is fine,
# that's not our code.
return len(
set(reactor.getReaders()) - set(reactor._internalReaders)
) + len(reactor.getWriters())
for i in range(timeout * 1000):
# There's a single DelayedCall for AsynchronousDeferredRunTest's
# timeout...
if (len(reactor.getDelayedCalls()) < 2 and num_fds() == 0):
break
await deferLater(reactor, 0.001)
return value
class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
# If set to True, use Foolscap for storage protocol. If set to False, HTTP
# will be used when possible. If set to None, this suggests a bug in the
# test code.
FORCE_FOOLSCAP_FOR_STORAGE : Optional[bool] = None
def setUp(self):
self._http_client_pools = []
http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool)
self.addCleanup(http_client.StorageClient.stop_test_mode)
self.port_assigner = SameProcessStreamEndpointAssigner()
self.port_assigner.setUp()
self.addCleanup(self.port_assigner.tearDown)
@ -652,10 +692,35 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
self.sparent = service.MultiService()
self.sparent.startService()
def _got_new_http_connection_pool(self, pool):
# Register the pool for shutdown later:
self._http_client_pools.append(pool)
# Disable retries:
pool.retryAutomatically = False
# Make a much more aggressive timeout for connections, we're connecting
# locally after all... and also make sure it's lower than the delay we
# add in tearDown, to prevent dirty reactor issues.
getConnection = pool.getConnection
def getConnectionWithTimeout(*args, **kwargs):
d = getConnection(*args, **kwargs)
d.addTimeout(1, reactor)
return d
pool.getConnection = getConnectionWithTimeout
def close_idle_http_connections(self):
"""Close all HTTP client connections that are just hanging around."""
return defer.gatherResults(
[pool.closeCachedConnections() for pool in self._http_client_pools]
)
def tearDown(self):
log.msg("shutting down SystemTest services")
d = self.sparent.stopService()
d.addBoth(flush_but_dont_ignore)
d.addBoth(lambda x: self.close_idle_http_connections().addCallback(lambda _: x))
d.addBoth(spin_until_cleanup_done)
return d
def getdir(self, subdir):
@ -714,21 +779,31 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
:return: A ``Deferred`` that fires when the nodes have connected to
each other.
"""
self.assertIn(
self.FORCE_FOOLSCAP_FOR_STORAGE, (True, False),
"You forgot to set FORCE_FOOLSCAP_FOR_STORAGE on {}".format(self.__class__)
)
self.numclients = NUMCLIENTS
self.introducer = yield self._create_introducer()
self.add_service(self.introducer)
self.introweb_url = self._get_introducer_web()
yield self._set_up_client_nodes()
yield self._set_up_client_nodes(self.FORCE_FOOLSCAP_FOR_STORAGE)
native_server = next(iter(self.clients[0].storage_broker.get_known_servers()))
if self.FORCE_FOOLSCAP_FOR_STORAGE:
expected_storage_server_class = NativeStorageServer
else:
expected_storage_server_class = HTTPNativeStorageServer
self.assertIsInstance(native_server, expected_storage_server_class)
@inlineCallbacks
def _set_up_client_nodes(self):
def _set_up_client_nodes(self, force_foolscap):
q = self.introducer
self.introducer_furl = q.introducer_url
self.clients = []
basedirs = []
for i in range(self.numclients):
basedirs.append((yield self._set_up_client_node(i)))
basedirs.append((yield self._set_up_client_node(i, force_foolscap)))
# start clients[0], wait for it's tub to be ready (at which point it
# will have registered the helper furl).
@ -761,7 +836,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
# and the helper-using webport
self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
def _generate_config(self, which, basedir):
def _generate_config(self, which, basedir, force_foolscap=False):
config = {}
allclients = set(range(self.numclients))
@ -791,6 +866,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
sethelper = partial(setconf, config, which, "helper")
setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,))
setconf(config, which, "storage", "force_foolscap", str(force_foolscap))
tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor)
setnode("tub.port", tub_port_endpoint)
@ -808,17 +884,16 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
" furl: %s\n") % self.introducer_furl
iyaml_fn = os.path.join(basedir, "private", "introducers.yaml")
fileutil.write(iyaml_fn, iyaml)
return _render_config(config)
def _set_up_client_node(self, which):
def _set_up_client_node(self, which, force_foolscap):
basedir = self.getdir("client%d" % (which,))
fileutil.make_dirs(os.path.join(basedir, "private"))
if len(SYSTEM_TEST_CERTS) > (which + 1):
f = open(os.path.join(basedir, "private", "node.pem"), "w")
f.write(SYSTEM_TEST_CERTS[which + 1])
f.close()
config = self._generate_config(which, basedir)
config = self._generate_config(which, basedir, force_foolscap)
fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
return basedir

28
src/allmydata/test/data/openssl-rsa-2048.txt Normal file
View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDF1MeXulDWFO05
YXCh8aqNc1dS1ddJRzsti4BOWuDOepUc0oCaSIcC5aR7XJ+vhX7a02mTIwvLcuEH
8sxx0BJU4jCDpRI6aAqaKJxwZx1e6AcVFJDl7vzymhvWhqHuKh0jTvwM2zONWTwV
V8m2PbDdxu0Prwdx+Mt2sDT6xHEhJj5fI/GUDUEdkhLJF6DQSulFRqqd0qP7qcI9
fSHZbM7MywfzqFUe8J1+tk4fBh2v7gNzN1INpzh2mDtLPAtxr4ZPtEb/0D0U4PsP
CniOHP0U8sF3VY0+K5qoCQr92cLRJvT/vLpQGVNUTFdFrtbqDoFxUCyEH4FUqRDX
2mVrPo2xAgMBAAECggEAA0Ev1y5/1NTPbgytBeIIH3d+v9hwKDbHecVoMwnOVeFJ
BZpONrOToovhAc1NXH2wj4SvwYWfpJ1HR9piDAuLeKlnuUu4ffzfE0gQok4E+v4r
2yg9ZcYBs/NOetAYVwbq960tiv/adFRr71E0WqbfS3fBx8q2L3Ujkkhd98PudUhQ
izbrTvkT7q00OPCWGwgWepMlLEowUWwZehGI0MlbONg7SbRraZZmG586Iy0tpC3e
AM7wC1/ORzFqcRgTIxXizQ5RHL7S0OQPLhbEJbuwPonNjze3p0EP4wNBELZTaVOd
xeA22Py4Bh/d1q3aEgbwR7tLyA8YfEzshTaY6oV8AQKBgQD0uFo8pyWk0AWXfjzn
jV4yYyPWy8pJA6YfAJAST8m7B/JeYgGlfHxTlNZiB40DsJq08tOZv3HAubgMpFIa
reuDxPqo6/Quwdy4Syu+AFhY48KIuwuoegG/L+5qcQLE69r1w71ZV6wUvLmXYX2I
Y6nYz+OdpD1JrMIr6Js60XURsQKBgQDO8yWl7ufIDKMbQpbs0PgUQsH4FtzGcP4J
j/7/8GfhKYt6rPsrojPHUbAi1+25xBVOuhm0Zx2ku2t+xPIMJoS+15EcER1Z2iHZ
Zci9UGpJpUxGcUhG7ETF1HZv0xKHcEOl9eIIOcAP9Vd9DqnGk85gy6ti6MHe/5Tn
IMD36OQ8AQKBgQDwqE7NMM67KnslRNaeG47T3F0FQbm3XehCuqnz6BUJYcI+gQD/
fdFB3K+LDcPmKgmqAtaGbxdtoPXXMM0xQXHHTrH15rxmMu1dK0dj/TDkkW7gSZko
YHtRSdCbSnGfuBXG9GxD7QzkA8g7j3sE4oXIGoDLqRVAW61DwubMy+jlsQKBgGNB
+Zepi1/Gt+BWQt8YpzPIhRIBnShMf3uEphCJdLlo3K4dE2btKBp8UpeTq0CDDJky
5ytAndYp0jf+K/2p59dEuyOUDdjPp5aGnA446JGkB35tzPW/Uoj0C049FVEChl+u
HBhH4peE285uXv2QXNbOOMh6zKmxOfDVI9iDyhwBAoGBAIXq2Ar0zDXXaL3ncEKo
pXt9BZ8OpJo2pvB1t2VPePOwEQ0wdT+H62fKNY47NiF9+LyS541/ps5Qhv6AmiKJ
Z7I0Vb6+sxQljYH/LNW+wc2T/pIAi/7sNcmnlBtZfoVwt99bk2CyoRALPLWHYCkh
c7Tty2bZzDZy6aCX+FGRt5N/
-----END PRIVATE KEY-----

11
src/allmydata/test/mutable/test_filenode.py
View File

@ -30,6 +30,7 @@ from allmydata.mutable.publish import MutableData
from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \
StoppingConsumer, ImmediatelyStoppingConsumer
from .. import common_util as testutil
from ...crypto.rsa import create_signing_keypair
from .util import (
FakeStorage,
make_nodemaker_with_peers,
@ -65,6 +66,16 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin):
d.addCallback(_created)
return d
async def test_create_with_keypair(self):
"""
An SDMF can be created using a given keypair.
"""
(priv, pub) = create_signing_keypair(2048)
node = await self.nodemaker.create_mutable_file(keypair=(pub, priv))
self.assertThat(
(node.get_privkey(), node.get_pubkey()),
Equals((priv, pub)),
)
def test_create_mdmf(self):
d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)

558
src/allmydata/test/mutable/test_version.py
View File

@ -1,19 +1,12 @@
"""
Ported to Python 3.
Tests related to the way ``allmydata.mutable`` handles different versions
of data for an object.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from io import StringIO
import os
from six.moves import cStringIO as StringIO
from typing import Optional
from twisted.internet import defer
from ..common import AsyncTestCase
from testtools.matchers import (
Equals,
@ -47,343 +40,268 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
self.small_data = b"test data" * 10 # 90 B; SDMF
def do_upload_mdmf(self, data=None):
async def do_upload_mdmf(self, data: Optional[bytes] = None) -> MutableFileNode:
if data is None:
data = self.data
d = self.nm.create_mutable_file(MutableData(data),
version=MDMF_VERSION)
def _then(n):
self.assertThat(n, IsInstance(MutableFileNode))
self.assertThat(n._protocol_version, Equals(MDMF_VERSION))
self.mdmf_node = n
return n
d.addCallback(_then)
return d
n = await self.nm.create_mutable_file(MutableData(data),
version=MDMF_VERSION)
self.assertThat(n, IsInstance(MutableFileNode))
self.assertThat(n._protocol_version, Equals(MDMF_VERSION))
self.mdmf_node = n
return n
def do_upload_sdmf(self, data=None):
async def do_upload_sdmf(self, data: Optional[bytes] = None) -> MutableFileNode:
if data is None:
data = self.small_data
d = self.nm.create_mutable_file(MutableData(data))
def _then(n):
self.assertThat(n, IsInstance(MutableFileNode))
self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
self.sdmf_node = n
return n
d.addCallback(_then)
return d
n = await self.nm.create_mutable_file(MutableData(data))
self.assertThat(n, IsInstance(MutableFileNode))
self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
self.sdmf_node = n
return n
def do_upload_empty_sdmf(self):
d = self.nm.create_mutable_file(MutableData(b""))
def _then(n):
self.assertThat(n, IsInstance(MutableFileNode))
self.sdmf_zero_length_node = n
self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
return n
d.addCallback(_then)
return d
async def do_upload_empty_sdmf(self) -> MutableFileNode:
n = await self.nm.create_mutable_file(MutableData(b""))
self.assertThat(n, IsInstance(MutableFileNode))
self.sdmf_zero_length_node = n
self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
return n
def do_upload(self):
d = self.do_upload_mdmf()
d.addCallback(lambda ign: self.do_upload_sdmf())
return d
async def do_upload(self) -> MutableFileNode:
await self.do_upload_mdmf()
return await self.do_upload_sdmf()
def test_debug(self):
d = self.do_upload_mdmf()
def _debug(n):
fso = debug.FindSharesOptions()
storage_index = base32.b2a(n.get_storage_index())
fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3
fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir)))
for (i,ss,storedir)
in self.iterate_servers()]
fso.stdout = StringIO()
fso.stderr = StringIO()
debug.find_shares(fso)
sharefiles = fso.stdout.getvalue().splitlines()
expected = self.nm.default_encoding_parameters["n"]
self.assertThat(sharefiles, HasLength(expected))
async def test_debug(self) -> None:
n = await self.do_upload_mdmf()
fso = debug.FindSharesOptions()
storage_index = base32.b2a(n.get_storage_index())
fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3
fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir)))
for (i,ss,storedir)
in self.iterate_servers()]
fso.stdout = StringIO()
fso.stderr = StringIO()
debug.find_shares(fso)
sharefiles = fso.stdout.getvalue().splitlines()
expected = self.nm.default_encoding_parameters["n"]
self.assertThat(sharefiles, HasLength(expected))
do = debug.DumpOptions()
do["filename"] = sharefiles[0]
do.stdout = StringIO()
debug.dump_share(do)
output = do.stdout.getvalue()
lines = set(output.splitlines())
self.assertTrue("Mutable slot found:" in lines, output)
self.assertTrue(" share_type: MDMF" in lines, output)
self.assertTrue(" num_extra_leases: 0" in lines, output)
self.assertTrue(" MDMF contents:" in lines, output)
self.assertTrue(" seqnum: 1" in lines, output)
self.assertTrue(" required_shares: 3" in lines, output)
self.assertTrue(" total_shares: 10" in lines, output)
self.assertTrue(" segsize: 131073" in lines, output)
self.assertTrue(" datalen: %d" % len(self.data) in lines, output)
vcap = str(n.get_verify_cap().to_string(), "utf-8")
self.assertTrue(" verify-cap: %s" % vcap in lines, output)
cso = debug.CatalogSharesOptions()
cso.nodedirs = fso.nodedirs
cso.stdout = StringIO()
cso.stderr = StringIO()
debug.catalog_shares(cso)
shares = cso.stdout.getvalue().splitlines()
oneshare = shares[0] # all shares should be MDMF
self.failIf(oneshare.startswith("UNKNOWN"), oneshare)
self.assertTrue(oneshare.startswith("MDMF"), oneshare)
fields = oneshare.split()
self.assertThat(fields[0], Equals("MDMF"))
self.assertThat(fields[1].encode("ascii"), Equals(storage_index))
self.assertThat(fields[2], Equals("3/10"))
self.assertThat(fields[3], Equals("%d" % len(self.data)))
self.assertTrue(fields[4].startswith("#1:"), fields[3])
# the rest of fields[4] is the roothash, which depends upon
# encryption salts and is not constant. fields[5] is the
# remaining time on the longest lease, which is timing dependent.
# The rest of the line is the quoted pathname to the share.
d.addCallback(_debug)
return d
do = debug.DumpOptions()
do["filename"] = sharefiles[0]
do.stdout = StringIO()
debug.dump_share(do)
output = do.stdout.getvalue()
lines = set(output.splitlines())
self.assertTrue("Mutable slot found:" in lines, output)
self.assertTrue(" share_type: MDMF" in lines, output)
self.assertTrue(" num_extra_leases: 0" in lines, output)
self.assertTrue(" MDMF contents:" in lines, output)
self.assertTrue(" seqnum: 1" in lines, output)
self.assertTrue(" required_shares: 3" in lines, output)
self.assertTrue(" total_shares: 10" in lines, output)
self.assertTrue(" segsize: 131073" in lines, output)
self.assertTrue(" datalen: %d" % len(self.data) in lines, output)
vcap = str(n.get_verify_cap().to_string(), "utf-8")
self.assertTrue(" verify-cap: %s" % vcap in lines, output)
cso = debug.CatalogSharesOptions()
cso.nodedirs = fso.nodedirs
cso.stdout = StringIO()
cso.stderr = StringIO()
debug.catalog_shares(cso)
shares = cso.stdout.getvalue().splitlines()
oneshare = shares[0] # all shares should be MDMF
self.failIf(oneshare.startswith("UNKNOWN"), oneshare)
self.assertTrue(oneshare.startswith("MDMF"), oneshare)
fields = oneshare.split()
self.assertThat(fields[0], Equals("MDMF"))
self.assertThat(fields[1].encode("ascii"), Equals(storage_index))
self.assertThat(fields[2], Equals("3/10"))
self.assertThat(fields[3], Equals("%d" % len(self.data)))
self.assertTrue(fields[4].startswith("#1:"), fields[3])
# the rest of fields[4] is the roothash, which depends upon
# encryption salts and is not constant. fields[5] is the
# remaining time on the longest lease, which is timing dependent.
# The rest of the line is the quoted pathname to the share.
async def test_get_sequence_number(self) -> None:
await self.do_upload()
bv = await self.mdmf_node.get_best_readable_version()
self.assertThat(bv.get_sequence_number(), Equals(1))
bv = await self.sdmf_node.get_best_readable_version()
self.assertThat(bv.get_sequence_number(), Equals(1))
def test_get_sequence_number(self):
d = self.do_upload()
d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version())
d.addCallback(lambda bv:
self.assertThat(bv.get_sequence_number(), Equals(1)))
d.addCallback(lambda ignored:
self.sdmf_node.get_best_readable_version())
d.addCallback(lambda bv:
self.assertThat(bv.get_sequence_number(), Equals(1)))
# Now update. The sequence number in both cases should be 1 in
# both cases.
def _do_update(ignored):
new_data = MutableData(b"foo bar baz" * 100000)
new_small_data = MutableData(b"foo bar baz" * 10)
d1 = self.mdmf_node.overwrite(new_data)
d2 = self.sdmf_node.overwrite(new_small_data)
dl = gatherResults([d1, d2])
return dl
d.addCallback(_do_update)
d.addCallback(lambda ignored:
self.mdmf_node.get_best_readable_version())
d.addCallback(lambda bv:
self.assertThat(bv.get_sequence_number(), Equals(2)))
d.addCallback(lambda ignored:
self.sdmf_node.get_best_readable_version())
d.addCallback(lambda bv:
self.assertThat(bv.get_sequence_number(), Equals(2)))
return d
new_data = MutableData(b"foo bar baz" * 100000)
new_small_data = MutableData(b"foo bar baz" * 10)
d1 = self.mdmf_node.overwrite(new_data)
d2 = self.sdmf_node.overwrite(new_small_data)
await gatherResults([d1, d2])
bv = await self.mdmf_node.get_best_readable_version()
self.assertThat(bv.get_sequence_number(), Equals(2))
bv = await self.sdmf_node.get_best_readable_version()
self.assertThat(bv.get_sequence_number(), Equals(2))
def test_cap_after_upload(self):
async def test_cap_after_upload(self) -> None:
# If we create a new mutable file and upload things to it, and
# it's an MDMF file, we should get an MDMF cap back from that
# file and should be able to use that.
# That's essentially what MDMF node is, so just check that.
d = self.do_upload_mdmf()
def _then(ign):
mdmf_uri = self.mdmf_node.get_uri()
cap = uri.from_string(mdmf_uri)
self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI))
readonly_mdmf_uri = self.mdmf_node.get_readonly_uri()
cap = uri.from_string(readonly_mdmf_uri)
self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI))
d.addCallback(_then)
return d
await self.do_upload_mdmf()
mdmf_uri = self.mdmf_node.get_uri()
cap = uri.from_string(mdmf_uri)
self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI))
readonly_mdmf_uri = self.mdmf_node.get_readonly_uri()
cap = uri.from_string(readonly_mdmf_uri)
self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI))
def test_mutable_version(self):
async def test_mutable_version(self) -> None:
# assert that getting parameters from the IMutableVersion object
# gives us the same data as getting them from the filenode itself
d = self.do_upload()
d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version())
def _check_mdmf(bv):
n = self.mdmf_node
self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
self.assertFalse(bv.is_readonly())
d.addCallback(_check_mdmf)
d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version())
def _check_sdmf(bv):
n = self.sdmf_node
self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
self.assertFalse(bv.is_readonly())
d.addCallback(_check_sdmf)
return d
await self.do_upload()
bv = await self.mdmf_node.get_best_mutable_version()
n = self.mdmf_node
self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
self.assertFalse(bv.is_readonly())
bv = await self.sdmf_node.get_best_mutable_version()
n = self.sdmf_node
self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
self.assertFalse(bv.is_readonly())
def test_get_readonly_version(self):
d = self.do_upload()
d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version())
d.addCallback(lambda bv: self.assertTrue(bv.is_readonly()))
async def test_get_readonly_version(self) -> None:
await self.do_upload()
bv = await self.mdmf_node.get_best_readable_version()
self.assertTrue(bv.is_readonly())
# Attempting to get a mutable version of a mutable file from a
# filenode initialized with a readcap should return a readonly
# version of that same node.
d.addCallback(lambda ign: self.mdmf_node.get_readonly())
d.addCallback(lambda ro: ro.get_best_mutable_version())
d.addCallback(lambda v: self.assertTrue(v.is_readonly()))
ro = self.mdmf_node.get_readonly()
v = await ro.get_best_mutable_version()
self.assertTrue(v.is_readonly())
d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version())
d.addCallback(lambda bv: self.assertTrue(bv.is_readonly()))
bv = await self.sdmf_node.get_best_readable_version()
self.assertTrue(bv.is_readonly())
d.addCallback(lambda ign: self.sdmf_node.get_readonly())
d.addCallback(lambda ro: ro.get_best_mutable_version())
d.addCallback(lambda v: self.assertTrue(v.is_readonly()))
return d
ro = self.sdmf_node.get_readonly()
v = await ro.get_best_mutable_version()
self.assertTrue(v.is_readonly())
def test_toplevel_overwrite(self):
async def test_toplevel_overwrite(self) -> None:
new_data = MutableData(b"foo bar baz" * 100000)
new_small_data = MutableData(b"foo bar baz" * 10)
d = self.do_upload()
d.addCallback(lambda ign: self.mdmf_node.overwrite(new_data))
d.addCallback(lambda ignored:
self.mdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Equals(b"foo bar baz" * 100000)))
d.addCallback(lambda ignored:
self.sdmf_node.overwrite(new_small_data))
d.addCallback(lambda ignored:
self.sdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Equals(b"foo bar baz" * 10)))
return d
await self.do_upload()
await self.mdmf_node.overwrite(new_data)
data = await self.mdmf_node.download_best_version()
self.assertThat(data, Equals(b"foo bar baz" * 100000))
await self.sdmf_node.overwrite(new_small_data)
data = await self.sdmf_node.download_best_version()
self.assertThat(data, Equals(b"foo bar baz" * 10))
def test_toplevel_modify(self):
d = self.do_upload()
async def test_toplevel_modify(self) -> None:
await self.do_upload()
def modifier(old_contents, servermap, first_time):
return old_contents + b"modified"
d.addCallback(lambda ign: self.mdmf_node.modify(modifier))
d.addCallback(lambda ignored:
self.mdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Contains(b"modified")))
d.addCallback(lambda ignored:
self.sdmf_node.modify(modifier))
d.addCallback(lambda ignored:
self.sdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Contains(b"modified")))
return d
await self.mdmf_node.modify(modifier)
data = await self.mdmf_node.download_best_version()
self.assertThat(data, Contains(b"modified"))
await self.sdmf_node.modify(modifier)
data = await self.sdmf_node.download_best_version()
self.assertThat(data, Contains(b"modified"))
def test_version_modify(self):
async def test_version_modify(self) -> None:
# TODO: When we can publish multiple versions, alter this test
# to modify a version other than the best usable version, then
# test to see that the best recoverable version is that.
d = self.do_upload()
await self.do_upload()
def modifier(old_contents, servermap, first_time):
return old_contents + b"modified"
d.addCallback(lambda ign: self.mdmf_node.modify(modifier))
d.addCallback(lambda ignored:
self.mdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Contains(b"modified")))
d.addCallback(lambda ignored:
self.sdmf_node.modify(modifier))
d.addCallback(lambda ignored:
self.sdmf_node.download_best_version())
d.addCallback(lambda data:
self.assertThat(data, Contains(b"modified")))
return d
await self.mdmf_node.modify(modifier)
data = await self.mdmf_node.download_best_version()
self.assertThat(data, Contains(b"modified"))
await self.sdmf_node.modify(modifier)
data = await self.sdmf_node.download_best_version()
self.assertThat(data, Contains(b"modified"))
def test_download_version(self):
d = self.publish_multiple()
async def test_download_version(self) -> None:
await self.publish_multiple()
# We want to have two recoverable versions on the grid.
d.addCallback(lambda res:
self._set_versions({0:0,2:0,4:0,6:0,8:0,
1:1,3:1,5:1,7:1,9:1}))
self._set_versions({0:0,2:0,4:0,6:0,8:0,
1:1,3:1,5:1,7:1,9:1})
# Now try to download each version. We should get the plaintext
# associated with that version.
d.addCallback(lambda ignored:
self._fn.get_servermap(mode=MODE_READ))
def _got_servermap(smap):
versions = smap.recoverable_versions()
assert len(versions) == 2
smap = await self._fn.get_servermap(mode=MODE_READ)
versions = smap.recoverable_versions()
assert len(versions) == 2
self.servermap = smap
self.version1, self.version2 = versions
assert self.version1 != self.version2
self.servermap = smap
self.version1, self.version2 = versions
assert self.version1 != self.version2
self.version1_seqnum = self.version1[0]
self.version2_seqnum = self.version2[0]
self.version1_index = self.version1_seqnum - 1
self.version2_index = self.version2_seqnum - 1
self.version1_seqnum = self.version1[0]
self.version2_seqnum = self.version2[0]
self.version1_index = self.version1_seqnum - 1
self.version2_index = self.version2_seqnum - 1
d.addCallback(_got_servermap)
d.addCallback(lambda ignored:
self._fn.download_version(self.servermap, self.version1))
d.addCallback(lambda results:
self.assertThat(self.CONTENTS[self.version1_index],
Equals(results)))
d.addCallback(lambda ignored:
self._fn.download_version(self.servermap, self.version2))
d.addCallback(lambda results:
self.assertThat(self.CONTENTS[self.version2_index],
Equals(results)))
return d
results = await self._fn.download_version(self.servermap, self.version1)
self.assertThat(self.CONTENTS[self.version1_index],
Equals(results))
results = await self._fn.download_version(self.servermap, self.version2)
self.assertThat(self.CONTENTS[self.version2_index],
Equals(results))
def test_download_nonexistent_version(self):
d = self.do_upload_mdmf()
d.addCallback(lambda ign: self.mdmf_node.get_servermap(mode=MODE_WRITE))
def _set_servermap(servermap):
self.servermap = servermap
d.addCallback(_set_servermap)
d.addCallback(lambda ignored:
self.shouldFail(UnrecoverableFileError, "nonexistent version",
None,
self.mdmf_node.download_version, self.servermap,
"not a version"))
return d
async def test_download_nonexistent_version(self) -> None:
await self.do_upload_mdmf()
servermap = await self.mdmf_node.get_servermap(mode=MODE_WRITE)
await self.shouldFail(UnrecoverableFileError, "nonexistent version",
None,
self.mdmf_node.download_version, servermap,
"not a version")
def _test_partial_read(self, node, expected, modes, step):
d = node.get_best_readable_version()
async def _test_partial_read(self, node, expected, modes, step) -> None:
version = await node.get_best_readable_version()
for (name, offset, length) in modes:
d.addCallback(self._do_partial_read, name, expected, offset, length)
await self._do_partial_read(version, name, expected, offset, length)
# then read the whole thing, but only a few bytes at a time, and see
# that the results are what we expect.
def _read_data(version):
c = consumer.MemoryConsumer()
d2 = defer.succeed(None)
for i in range(0, len(expected), step):
d2.addCallback(lambda ignored, i=i: version.read(c, i, step))
d2.addCallback(lambda ignored:
self.assertThat(expected, Equals(b"".join(c.chunks))))
return d2
d.addCallback(_read_data)
return d
def _do_partial_read(self, version, name, expected, offset, length):
c = consumer.MemoryConsumer()
d = version.read(c, offset, length)
for i in range(0, len(expected), step):
await version.read(c, i, step)
self.assertThat(expected, Equals(b"".join(c.chunks)))
async def _do_partial_read(self, version, name, expected, offset, length) -> None:
c = consumer.MemoryConsumer()
await version.read(c, offset, length)
if length is None:
expected_range = expected[offset:]
else:
expected_range = expected[offset:offset+length]
d.addCallback(lambda ignored: b"".join(c.chunks))
def _check(results):
if results != expected_range:
print("read([%d]+%s) got %d bytes, not %d" % \
(offset, length, len(results), len(expected_range)))
print("got: %s ... %s" % (results[:20], results[-20:]))
print("exp: %s ... %s" % (expected_range[:20], expected_range[-20:]))
self.fail("results[%s] != expected_range" % name)
return version # daisy-chained to next call
d.addCallback(_check)
return d
results = b"".join(c.chunks)
if results != expected_range:
print("read([%d]+%s) got %d bytes, not %d" % \
(offset, length, len(results), len(expected_range)))
print("got: %r ... %r" % (results[:20], results[-20:]))
print("exp: %r ... %r" % (expected_range[:20], expected_range[-20:]))
self.fail("results[%s] != expected_range" % name)
def test_partial_read_mdmf_0(self):
async def test_partial_read_mdmf_0(self) -> None:
data = b""
d = self.do_upload_mdmf(data=data)
result = await self.do_upload_mdmf(data=data)
modes = [("all1", 0,0),
("all2", 0,None),
]
d.addCallback(self._test_partial_read, data, modes, 1)
return d
await self._test_partial_read(result, data, modes, 1)
def test_partial_read_mdmf_large(self):
async def test_partial_read_mdmf_large(self) -> None:
segment_boundary = mathutil.next_multiple(128 * 1024, 3)
modes = [("start_on_segment_boundary", segment_boundary, 50),
("ending_one_byte_after_segment_boundary", segment_boundary-50, 51),
@ -393,20 +311,18 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
("complete_file1", 0, len(self.data)),
("complete_file2", 0, None),
]
d = self.do_upload_mdmf()
d.addCallback(self._test_partial_read, self.data, modes, 10000)
return d
result = await self.do_upload_mdmf()
await self._test_partial_read(result, self.data, modes, 10000)
def test_partial_read_sdmf_0(self):
async def test_partial_read_sdmf_0(self) -> None:
data = b""
modes = [("all1", 0,0),
("all2", 0,None),
]
d = self.do_upload_sdmf(data=data)
d.addCallback(self._test_partial_read, data, modes, 1)
return d
result = await self.do_upload_sdmf(data=data)
await self._test_partial_read(result, data, modes, 1)
def test_partial_read_sdmf_2(self):
async def test_partial_read_sdmf_2(self) -> None:
data = b"hi"
modes = [("one_byte", 0, 1),
("last_byte", 1, 1),
@ -414,11 +330,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
("complete_file", 0, 2),
("complete_file2", 0, None),
]
d = self.do_upload_sdmf(data=data)
d.addCallback(self._test_partial_read, data, modes, 1)
return d
result = await self.do_upload_sdmf(data=data)
await self._test_partial_read(result, data, modes, 1)
def test_partial_read_sdmf_90(self):
async def test_partial_read_sdmf_90(self) -> None:
modes = [("start_at_middle", 50, 40),
("start_at_middle2", 50, None),
("zero_length_at_start", 0, 0),
@ -427,11 +342,10 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
("complete_file1", 0, None),
("complete_file2", 0, 90),
]
d = self.do_upload_sdmf()
d.addCallback(self._test_partial_read, self.small_data, modes, 10)
return d
result = await self.do_upload_sdmf()
await self._test_partial_read(result, self.small_data, modes, 10)
def test_partial_read_sdmf_100(self):
async def test_partial_read_sdmf_100(self) -> None:
data = b"test data "*10
modes = [("start_at_middle", 50, 50),
("start_at_middle2", 50, None),
@ -440,42 +354,30 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
("complete_file1", 0, 100),
("complete_file2", 0, None),
]
d = self.do_upload_sdmf(data=data)
d.addCallback(self._test_partial_read, data, modes, 10)
return d
result = await self.do_upload_sdmf(data=data)
await self._test_partial_read(result, data, modes, 10)
async def _test_read_and_download(self, node, expected) -> None:
version = await node.get_best_readable_version()
c = consumer.MemoryConsumer()
await version.read(c)
self.assertThat(expected, Equals(b"".join(c.chunks)))
def _test_read_and_download(self, node, expected):
d = node.get_best_readable_version()
def _read_data(version):
c = consumer.MemoryConsumer()
c2 = consumer.MemoryConsumer()
d2 = defer.succeed(None)
d2.addCallback(lambda ignored: version.read(c))
d2.addCallback(lambda ignored:
self.assertThat(expected, Equals(b"".join(c.chunks))))
c2 = consumer.MemoryConsumer()
await version.read(c2, offset=0, size=len(expected))
self.assertThat(expected, Equals(b"".join(c2.chunks)))
d2.addCallback(lambda ignored: version.read(c2, offset=0,
size=len(expected)))
d2.addCallback(lambda ignored:
self.assertThat(expected, Equals(b"".join(c2.chunks))))
return d2
d.addCallback(_read_data)
d.addCallback(lambda ignored: node.download_best_version())
d.addCallback(lambda data: self.assertThat(expected, Equals(data)))
return d
data = await node.download_best_version()
self.assertThat(expected, Equals(data))
def test_read_and_download_mdmf(self):
d = self.do_upload_mdmf()
d.addCallback(self._test_read_and_download, self.data)
return d
async def test_read_and_download_mdmf(self) -> None:
result = await self.do_upload_mdmf()
await self._test_read_and_download(result, self.data)
def test_read_and_download_sdmf(self):
d = self.do_upload_sdmf()
d.addCallback(self._test_read_and_download, self.small_data)
return d
async def test_read_and_download_sdmf(self) -> None:
result = await self.do_upload_sdmf()
await self._test_read_and_download(result, self.small_data)
def test_read_and_download_sdmf_zero_length(self):
d = self.do_upload_empty_sdmf()
d.addCallback(self._test_read_and_download, b"")
return d
async def test_read_and_download_sdmf_zero_length(self) -> None:
result = await self.do_upload_empty_sdmf()
await self._test_read_and_download(result, b"")

3
src/allmydata/test/test_dirnode.py
View File

@ -1619,7 +1619,8 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation
return defer.succeed(None)
class FakeNodeMaker(NodeMaker):
def create_mutable_file(self, contents=b"", keysize=None, version=None):
def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None):
assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs"
return defer.succeed(FakeMutableFile(contents))
class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573

90
src/allmydata/test/test_istorageserver.py
View File

@ -15,9 +15,8 @@ from typing import Set
from random import Random
from unittest import SkipTest
from twisted.internet.defer import inlineCallbacks, returnValue, succeed
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.task import Clock
from twisted.internet import reactor
from foolscap.api import Referenceable, RemoteException
# A better name for this would be IStorageClient...
@ -26,8 +25,6 @@ from allmydata.interfaces import IStorageServer
from .common_system import SystemTestMixin
from .common import AsyncTestCase
from allmydata.storage.server import StorageServer # not a IStorageServer!!
from allmydata.storage.http_client import StorageClient
from allmydata.storage_client import _HTTPStorageServer
# Use random generator with known seed, so results are reproducible if tests
@ -439,6 +436,17 @@ class IStorageServerImmutableAPIsTestsMixin(object):
b"immutable", storage_index, 0, b"ono"
)
@inlineCallbacks
def test_advise_corrupt_share_unknown_share_number(self):
"""
Calling ``advise_corrupt_share()`` on an immutable share, with an
unknown share number, does not result in error.
"""
storage_index, _, _ = yield self.create_share()
yield self.storage_client.advise_corrupt_share(
b"immutable", storage_index, 999, b"ono"
)
@inlineCallbacks
def test_allocate_buckets_creates_lease(self):
"""
@ -908,6 +916,19 @@ class IStorageServerMutableAPIsTestsMixin(object):
b"mutable", storage_index, 0, b"ono"
)
@inlineCallbacks
def test_advise_corrupt_share_unknown_share_number(self):
"""
Calling ``advise_corrupt_share()`` on a mutable share with an unknown
share number does not result in error (other behavior is opaque at this
level of abstraction).
"""
secrets, storage_index = yield self.create_slot()
yield self.storage_client.advise_corrupt_share(
b"mutable", storage_index, 999, b"ono"
)
@inlineCallbacks
def test_STARAW_create_lease(self):
"""
@ -1023,7 +1044,10 @@ class _SharedMixin(SystemTestMixin):
SKIP_TESTS = set() # type: Set[str]
def _get_istorage_server(self):
raise NotImplementedError("implement in subclass")
native_server = next(iter(self.clients[0].storage_broker.get_known_servers()))
client = native_server.get_storage_server()
self.assertTrue(IStorageServer.providedBy(client))
return client
@inlineCallbacks
def setUp(self):
@ -1046,7 +1070,7 @@ class _SharedMixin(SystemTestMixin):
self._clock = Clock()
self._clock.advance(123456)
self.server._clock = self._clock
self.storage_client = yield self._get_istorage_server()
self.storage_client = self._get_istorage_server()
def fake_time(self):
"""Return the current fake, test-controlled, time."""
@ -1062,51 +1086,29 @@ class _SharedMixin(SystemTestMixin):
yield SystemTestMixin.tearDown(self)
class _FoolscapMixin(_SharedMixin):
"""Run tests on Foolscap version of ``IStorageServer``."""
def _get_native_server(self):
return next(iter(self.clients[0].storage_broker.get_known_servers()))
def _get_istorage_server(self):
client = self._get_native_server().get_storage_server()
self.assertTrue(IStorageServer.providedBy(client))
return succeed(client)
class _HTTPMixin(_SharedMixin):
"""Run tests on the HTTP version of ``IStorageServer``."""
def _get_istorage_server(self):
nurl = list(self.clients[0].storage_nurls)[0]
# Create HTTP client with non-persistent connections, so we don't leak
# state across tests:
client: IStorageServer = _HTTPStorageServer.from_http_client(
StorageClient.from_nurl(nurl, reactor, persistent=False)
)
self.assertTrue(IStorageServer.providedBy(client))
return succeed(client)
class FoolscapSharedAPIsTests(
_FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase
):
"""Foolscap-specific tests for shared ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = True
class HTTPSharedAPIsTests(
_HTTPMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase
):
"""HTTP-specific tests for shared ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = False
class FoolscapImmutableAPIsTests(
_FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase
):
"""Foolscap-specific tests for immutable ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = True
def test_disconnection(self):
"""
If we disconnect in the middle of writing to a bucket, all data is
@ -1129,23 +1131,29 @@ class FoolscapImmutableAPIsTests(
"""
current = self.storage_client
yield self.bounce_client(0)
self.storage_client = self._get_native_server().get_storage_server()
self.storage_client = self._get_istorage_server()
assert self.storage_client is not current
class HTTPImmutableAPIsTests(
_HTTPMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase
):
"""HTTP-specific tests for immutable ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = False
class FoolscapMutableAPIsTests(
_FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase
):
"""Foolscap-specific tests for mutable ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = True
class HTTPMutableAPIsTests(
_HTTPMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase
_SharedMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase
):
"""HTTP-specific tests for mutable ``IStorageServer`` APIs."""
FORCE_FOOLSCAP_FOR_STORAGE = False

198
src/allmydata/test/test_pipeline.py
View File

@ -1,198 +0,0 @@
"""
Tests for allmydata.util.pipeline.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import gc
from twisted.internet import defer
from twisted.trial import unittest
from twisted.python import log
from twisted.python.failure import Failure
from allmydata.util import pipeline
class Pipeline(unittest.TestCase):
def pause(self, *args, **kwargs):
d = defer.Deferred()
self.calls.append( (d, args, kwargs) )
return d
def failUnlessCallsAre(self, expected):
#print(self.calls)
#print(expected)
self.failUnlessEqual(len(self.calls), len(expected), self.calls)
for i,c in enumerate(self.calls):
self.failUnlessEqual(c[1:], expected[i], str(i))
def test_basic(self):
self.calls = []
finished = []
p = pipeline.Pipeline(100)
d = p.flush() # fires immediately
d.addCallbacks(finished.append, log.err)
self.failUnlessEqual(len(finished), 1)
finished = []
d = p.add(10, self.pause, "one")
# the call should start right away, and our return Deferred should
# fire right away
d.addCallbacks(finished.append, log.err)
self.failUnlessEqual(len(finished), 1)
self.failUnlessEqual(finished[0], None)
self.failUnlessCallsAre([ ( ("one",) , {} ) ])
self.failUnlessEqual(p.gauge, 10)
# pipeline: [one]
finished = []
d = p.add(20, self.pause, "two", kw=2)
# pipeline: [one, two]
# the call and the Deferred should fire right away
d.addCallbacks(finished.append, log.err)
self.failUnlessEqual(len(finished), 1)
self.failUnlessEqual(finished[0], None)
self.failUnlessCallsAre([ ( ("one",) , {} ),
( ("two",) , {"kw": 2} ),
])
self.failUnlessEqual(p.gauge, 30)
self.calls[0][0].callback("one-result")
# pipeline: [two]
self.failUnlessEqual(p.gauge, 20)
finished = []
d = p.add(90, self.pause, "three", "posarg1")
# pipeline: [two, three]
flushed = []
fd = p.flush()
fd.addCallbacks(flushed.append, log.err)
self.failUnlessEqual(flushed, [])
# the call will be made right away, but the return Deferred will not,
# because the pipeline is now full.
d.addCallbacks(finished.append, log.err)
self.failUnlessEqual(len(finished), 0)
self.failUnlessCallsAre([ ( ("one",) , {} ),
( ("two",) , {"kw": 2} ),
( ("three", "posarg1"), {} ),
])
self.failUnlessEqual(p.gauge, 110)
self.failUnlessRaises(pipeline.SingleFileError, p.add, 10, self.pause)
# retiring either call will unblock the pipeline, causing the #3
# Deferred to fire
self.calls[2][0].callback("three-result")
# pipeline: [two]
self.failUnlessEqual(len(finished), 1)
self.failUnlessEqual(finished[0], None)
self.failUnlessEqual(flushed, [])
# retiring call#2 will finally allow the flush() Deferred to fire
self.calls[1][0].callback("two-result")
self.failUnlessEqual(len(flushed), 1)
def test_errors(self):
self.calls = []
p = pipeline.Pipeline(100)
d1 = p.add(200, self.pause, "one")
d2 = p.flush()
finished = []
d1.addBoth(finished.append)
self.failUnlessEqual(finished, [])
flushed = []
d2.addBoth(flushed.append)
self.failUnlessEqual(flushed, [])
self.calls[0][0].errback(ValueError("oops"))
self.failUnlessEqual(len(finished), 1)
f = finished[0]
self.failUnless(isinstance(f, Failure))
self.failUnless(f.check(pipeline.PipelineError))
self.failUnlessIn("PipelineError", str(f.value))
self.failUnlessIn("ValueError", str(f.value))
r = repr(f.value)
self.failUnless("ValueError" in r, r)
f2 = f.value.error
self.failUnless(f2.check(ValueError))
self.failUnlessEqual(len(flushed), 1)
f = flushed[0]
self.failUnless(isinstance(f, Failure))
self.failUnless(f.check(pipeline.PipelineError))
f2 = f.value.error
self.failUnless(f2.check(ValueError))
# now that the pipeline is in the failed state, any new calls will
# fail immediately
d3 = p.add(20, self.pause, "two")
finished = []
d3.addBoth(finished.append)
self.failUnlessEqual(len(finished), 1)
f = finished[0]
self.failUnless(isinstance(f, Failure))
self.failUnless(f.check(pipeline.PipelineError))
r = repr(f.value)
self.failUnless("ValueError" in r, r)
f2 = f.value.error
self.failUnless(f2.check(ValueError))
d4 = p.flush()
flushed = []
d4.addBoth(flushed.append)
self.failUnlessEqual(len(flushed), 1)
f = flushed[0]
self.failUnless(isinstance(f, Failure))
self.failUnless(f.check(pipeline.PipelineError))
f2 = f.value.error
self.failUnless(f2.check(ValueError))
def test_errors2(self):
self.calls = []
p = pipeline.Pipeline(100)
d1 = p.add(10, self.pause, "one")
d2 = p.add(20, self.pause, "two")
d3 = p.add(30, self.pause, "three")
d4 = p.flush()
# one call fails, then the second one succeeds: make sure
# ExpandableDeferredList tolerates the second one
flushed = []
d4.addBoth(flushed.append)
self.failUnlessEqual(flushed, [])
self.calls[0][0].errback(ValueError("oops"))
self.failUnlessEqual(len(flushed), 1)
f = flushed[0]
self.failUnless(isinstance(f, Failure))
self.failUnless(f.check(pipeline.PipelineError))
f2 = f.value.error
self.failUnless(f2.check(ValueError))
self.calls[1][0].callback("two-result")
self.calls[2][0].errback(ValueError("three-error"))
del d1,d2,d3,d4
gc.collect() # for PyPy

64
src/allmydata/test/test_runner.py
View File

@ -47,6 +47,9 @@ from twisted.internet.defer import (
inlineCallbacks,
DeferredList,
)
from twisted.internet.testing import (
MemoryReactorClock,
)
from twisted.python.filepath import FilePath
from allmydata.util import fileutil, pollmixin
from allmydata.util.encodingutil import unicode_to_argv
@ -60,6 +63,9 @@ import allmydata
from allmydata.scripts.runner import (
parse_options,
)
from allmydata.scripts.tahoe_run import (
on_stdin_close,
)
from .common import (
PIPE,
@ -624,6 +630,64 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
yield client_running
def _simulate_windows_stdin_close(stdio):
"""
on Unix we can just close all the readers, correctly "simulating"
a stdin close .. of course, Windows has to be difficult
"""
stdio.writeConnectionLost()
stdio.readConnectionLost()
class OnStdinCloseTests(SyncTestCase):
"""
Tests for on_stdin_close
"""
def test_close_called(self):
"""
our on-close method is called when stdin closes
"""
reactor = MemoryReactorClock()
called = []
def onclose():
called.append(True)
transport = on_stdin_close(reactor, onclose)
self.assertEqual(called, [])
if platform.isWindows():
_simulate_windows_stdin_close(transport)
else:
for reader in reactor.getReaders():
reader.loseConnection()
reactor.advance(1) # ProcessReader does a callLater(0, ..)
self.assertEqual(called, [True])
def test_exception_ignored(self):
"""
An exception from our on-close function is discarded.
"""
reactor = MemoryReactorClock()
called = []
def onclose():
called.append(True)
raise RuntimeError("unexpected error")
transport = on_stdin_close(reactor, onclose)
self.assertEqual(called, [])
if platform.isWindows():
_simulate_windows_stdin_close(transport)
else:
for reader in reactor.getReaders():
reader.loseConnection()
reactor.advance(1) # ProcessReader does a callLater(0, ..)
self.assertEqual(called, [True])
class PidFileLocking(SyncTestCase):
"""
Direct tests for allmydata.util.pid functions

47
src/allmydata/test/test_storage.py
View File

@ -3,14 +3,9 @@ Tests for allmydata.storage.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import native_str, PY2, bytes_to_native_str, bchr
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from future.utils import native_str, bytes_to_native_str, bchr
from six import ensure_str
from io import (
@ -59,7 +54,7 @@ from allmydata.storage.common import storage_index_to_dir, \
si_b2a, si_a2b
from allmydata.storage.lease import LeaseInfo
from allmydata.immutable.layout import WriteBucketProxy, WriteBucketProxy_v2, \
ReadBucketProxy
ReadBucketProxy, _WriteBuffer
from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \
LayoutInvalid, MDMFSIGNABLEHEADER, \
SIGNED_PREFIX, MDMFHEADER, \
@ -3746,3 +3741,39 @@ class LeaseInfoTests(SyncTestCase):
info.to_mutable_data(),
HasLength(info.mutable_size()),
)
class WriteBufferTests(SyncTestCase):
"""Tests for ``_WriteBuffer``."""
@given(
small_writes=strategies.lists(
strategies.binary(min_size=1, max_size=20),
min_size=10, max_size=20),
batch_size=strategies.integers(min_value=5, max_value=10)
)
def test_write_buffer(self, small_writes: list[bytes], batch_size: int):
"""
``_WriteBuffer`` coalesces small writes into bigger writes based on
the batch size.
"""
wb = _WriteBuffer(batch_size)
result = b""
for data in small_writes:
should_flush = wb.queue_write(data)
if should_flush:
flushed_offset, flushed_data = wb.flush()
self.assertEqual(flushed_offset, len(result))
# The flushed data is in batch sizes, or closest approximation
# given queued inputs:
self.assertTrue(batch_size <= len(flushed_data) < batch_size + len(data))
result += flushed_data
# Final flush:
remaining_length = wb.get_queued_bytes()
flushed_offset, flushed_data = wb.flush()
self.assertEqual(remaining_length, len(flushed_data))
self.assertEqual(flushed_offset, len(result))
result += flushed_data
self.assertEqual(result, b"".join(small_writes))

158
src/allmydata/test/test_storage_http.py
View File

@ -31,10 +31,13 @@ from klein import Klein
from hyperlink import DecodedURL
from collections_extended import RangeMap
from twisted.internet.task import Clock, Cooperator
from twisted.internet.interfaces import IReactorTime
from twisted.internet.defer import CancelledError, Deferred
from twisted.web import http
from twisted.web.http_headers import Headers
from werkzeug import routing
from werkzeug.exceptions import NotFound as WNotFound
from testtools.matchers import Equals
from .common import SyncTestCase
from ..storage.http_common import get_content_type, CBOR_MIME_TYPE
@ -245,6 +248,7 @@ def gen_bytes(length: int) -> bytes:
class TestApp(object):
"""HTTP API for testing purposes."""
clock: IReactorTime
_app = Klein()
_swissnum = SWISSNUM_FOR_TEST # Match what the test client is using
@ -266,6 +270,25 @@ class TestApp(object):
"""Return bytes to the given length using ``gen_bytes()``."""
return gen_bytes(length)
@_authorized_route(_app, set(), "/slowly_never_finish_result", methods=["GET"])
def slowly_never_finish_result(self, request, authorization):
"""
Send data immediately, after 59 seconds, after another 59 seconds, and then
never again, without finishing the response.
"""
request.write(b"a")
self.clock.callLater(59, request.write, b"b")
self.clock.callLater(59 + 59, request.write, b"c")
return Deferred()
@_authorized_route(_app, set(), "/die_unfinished", methods=["GET"])
def die(self, request, authorization):
"""
Dies half-way.
"""
request.transport.loseConnection()
return Deferred()
def result_of(d):
"""
@ -291,14 +314,25 @@ class CustomHTTPServerTests(SyncTestCase):
def setUp(self):
super(CustomHTTPServerTests, self).setUp()
StorageClient.start_test_mode(
lambda pool: self.addCleanup(pool.closeCachedConnections)
)
self.addCleanup(StorageClient.stop_test_mode)
# Could be a fixture, but will only be used in this test class so not
# going to bother:
self._http_server = TestApp()
treq = StubTreq(self._http_server._app.resource())
self.client = StorageClient(
DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST,
treq=StubTreq(self._http_server._app.resource()),
treq=treq,
# We're using a Treq private API to get the reactor, alas, but only
# in a test, so not going to worry about it too much. This would be
# fixed if https://github.com/twisted/treq/issues/226 were ever
# fixed.
clock=treq._agent._memoryReactor,
)
self._http_server.clock = self.client._clock
def test_authorization_enforcement(self):
"""
@ -346,7 +380,9 @@ class CustomHTTPServerTests(SyncTestCase):
)
self.assertEqual(
result_of(limited_content(response, at_least_length)).read(),
result_of(
limited_content(response, self._http_server.clock, at_least_length)
).read(),
gen_bytes(length),
)
@ -365,7 +401,52 @@ class CustomHTTPServerTests(SyncTestCase):
)
with self.assertRaises(ValueError):
result_of(limited_content(response, too_short))
result_of(limited_content(response, self._http_server.clock, too_short))
def test_limited_content_silence_causes_timeout(self):
"""
``http_client.limited_content() times out if it receives no data for 60
seconds.
"""
response = result_of(
self.client.request(
"GET",
"http://127.0.0.1/slowly_never_finish_result",
)
)
body_deferred = limited_content(response, self._http_server.clock, 4)
result = []
error = []
body_deferred.addCallbacks(result.append, error.append)
for i in range(59 + 59 + 60):
self.assertEqual((result, error), ([], []))
self._http_server.clock.advance(1)
# Push data between in-memory client and in-memory server:
self.client._treq._agent.flush()
# After 59 (second write) + 59 (third write) + 60 seconds (quiescent
# timeout) the limited_content() response times out.
self.assertTrue(error)
with self.assertRaises(CancelledError):
error[0].raiseException()
def test_limited_content_cancels_timeout_on_failed_response(self):
"""
If the response fails somehow, the timeout is still cancelled.
"""
response = result_of(
self.client.request(
"GET",
"http://127.0.0.1/die",
)
)
d = limited_content(response, self._http_server.clock, 4)
with self.assertRaises(ValueError):
result_of(d)
self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0)
class HttpTestFixture(Fixture):
@ -375,6 +456,10 @@ class HttpTestFixture(Fixture):
"""
def _setUp(self):
StorageClient.start_test_mode(
lambda pool: self.addCleanup(pool.closeCachedConnections)
)
self.addCleanup(StorageClient.stop_test_mode)
self.clock = Clock()
self.tempdir = self.useFixture(TempDir())
# The global Cooperator used by Twisted (a) used by pull producers in
@ -396,6 +481,7 @@ class HttpTestFixture(Fixture):
DecodedURL.from_text("http://127.0.0.1"),
SWISSNUM_FOR_TEST,
treq=self.treq,
clock=self.clock,
)
def result_of_with_flush(self, d):
@ -470,6 +556,20 @@ class GenericHTTPAPITests(SyncTestCase):
super(GenericHTTPAPITests, self).setUp()
self.http = self.useFixture(HttpTestFixture())
def test_missing_authentication(self) -> None:
"""
If nothing is given in the ``Authorization`` header at all an
``Unauthorized`` response is returned.
"""
client = StubTreq(self.http.http_server.get_resource())
response = self.http.result_of_with_flush(
client.request(
"GET",
"http://127.0.0.1/storage/v1/version",
),
)
self.assertThat(response.code, Equals(http.UNAUTHORIZED))
def test_bad_authentication(self):
"""
If the wrong swissnum is used, an ``Unauthorized`` response code is
@ -480,6 +580,7 @@ class GenericHTTPAPITests(SyncTestCase):
DecodedURL.from_text("http://127.0.0.1"),
b"something wrong",
treq=StubTreq(self.http.http_server.get_resource()),
clock=self.http.clock,
)
)
with assert_fails_with_http_code(self, http.UNAUTHORIZED):
@ -1100,18 +1201,42 @@ class MutableHTTPAPIsTests(SyncTestCase):
)
return storage_index, write_secret, lease_secret
def test_write_can_be_read(self):
def test_write_can_be_read_small_data(self):
"""
Small written data can be read using ``read_share_chunk``.
"""
self.write_can_be_read(b"abcdef")
def test_write_can_be_read_large_data(self):
"""
Large written data (50MB) can be read using ``read_share_chunk``.
"""
self.write_can_be_read(b"abcdefghij" * 5 * 1024 * 1024)
def write_can_be_read(self, data):
"""
Written data can be read using ``read_share_chunk``.
"""
storage_index, _, _ = self.create_upload()
data0 = self.http.result_of_with_flush(
self.mut_client.read_share_chunk(storage_index, 0, 1, 7)
lease_secret = urandom(32)
storage_index = urandom(16)
self.http.result_of_with_flush(
self.mut_client.read_test_write_chunks(
storage_index,
urandom(32),
lease_secret,
lease_secret,
{
0: TestWriteVectors(
write_vectors=[WriteVector(offset=0, data=data)]
),
},
[],
)
)
data1 = self.http.result_of_with_flush(
self.mut_client.read_share_chunk(storage_index, 1, 0, 8)
read_data = self.http.result_of_with_flush(
self.mut_client.read_share_chunk(storage_index, 0, 0, len(data))
)
self.assertEqual((data0, data1), (b"bcdef-0", b"abcdef-1"))
self.assertEqual(read_data, data)
def test_read_before_write(self):
"""In combo read/test/write operation, reads happen before writes."""
@ -1190,15 +1315,6 @@ class MutableHTTPAPIsTests(SyncTestCase):
b"aXYZef-0",
)
def test_too_large_write(self):
"""
Writing too large of a chunk results in a REQUEST ENTITY TOO LARGE http
error.
"""
with self.assertRaises(ClientException) as e:
self.create_upload(b"0123456789" * 1024 * 1024)
self.assertEqual(e.exception.code, http.REQUEST_ENTITY_TOO_LARGE)
def test_list_shares(self):
"""``list_shares()`` returns the shares for a given storage index."""
storage_index, _, _ = self.create_upload()
@ -1441,7 +1557,9 @@ class SharedImmutableMutableTestsMixin:
self.http.client.request(
"GET",
self.http.client.relative_url(
"/storage/v1/{}/{}/1".format(self.KIND, _encode_si(storage_index))
"/storage/v1/{}/{}/1".format(
self.KIND, _encode_si(storage_index)
)
),
headers=headers,
)

14
src/allmydata/test/test_storage_https.py
View File

@ -12,7 +12,7 @@ from cryptography import x509
from twisted.internet.endpoints import serverFromString
from twisted.internet import reactor
from twisted.internet.task import deferLater
from twisted.internet.defer import maybeDeferred
from twisted.web.server import Site
from twisted.web.static import Data
from twisted.web.client import Agent, HTTPConnectionPool, ResponseNeverReceived
@ -30,6 +30,7 @@ from ..storage.http_common import get_spki_hash
from ..storage.http_client import _StorageClientHTTPSPolicy
from ..storage.http_server import _TLSEndpointWrapper
from ..util.deferredutil import async_to_deferred
from .common_system import spin_until_cleanup_done
class HTTPSNurlTests(SyncTestCase):
@ -87,6 +88,10 @@ class PinningHTTPSValidation(AsyncTestCase):
self.addCleanup(self._port_assigner.tearDown)
return AsyncTestCase.setUp(self)
def tearDown(self):
d = maybeDeferred(AsyncTestCase.tearDown, self)
return d.addCallback(lambda _: spin_until_cleanup_done())
@asynccontextmanager
async def listen(self, private_key_path: FilePath, cert_path: FilePath):
"""
@ -107,9 +112,6 @@ class PinningHTTPSValidation(AsyncTestCase):
yield f"https://127.0.0.1:{listening_port.getHost().port}/"
finally:
await listening_port.stopListening()
# Make sure all server connections are closed :( No idea why this
# is necessary when it's not for IStorageServer HTTPS tests.
await deferLater(reactor, 0.01)
def request(self, url: str, expected_certificate: x509.Certificate):
"""
@ -198,10 +200,6 @@ class PinningHTTPSValidation(AsyncTestCase):
response = await self.request(url, certificate)
self.assertEqual(await response.content(), b"YOYODYNE")
# We keep getting TLSMemoryBIOProtocol being left around, so try harder
# to wait for it to finish.
await deferLater(reactor, 0.001)
# A potential attack to test is a private key that doesn't match the
# certificate... but OpenSSL (quite rightly) won't let you listen with that
# so I don't know how to test that! See

94
src/allmydata/test/test_system.py
View File

@ -34,7 +34,7 @@ from allmydata.util.encodingutil import quote_output, unicode_to_argv
from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.util.consumer import MemoryConsumer, download_to_data
from allmydata.interfaces import IDirectoryNode, IFileNode, \
NoSuchChildError, NoSharesError
NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION
from allmydata.monitor import Monitor
from allmydata.mutable.common import NotWriteableError
from allmydata.mutable import layout as mutable_layout
@ -117,11 +117,17 @@ class CountingDataUploadable(upload.Data):
class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
"""Foolscap integration-y tests."""
FORCE_FOOLSCAP_FOR_STORAGE = True
timeout = 180
@property
def basedir(self):
return "system/SystemTest/{}-foolscap-{}".format(
self.id().split(".")[-1], self.FORCE_FOOLSCAP_FOR_STORAGE
)
def test_connections(self):
self.basedir = "system/SystemTest/test_connections"
d = self.set_up_nodes()
self.extra_node = None
d.addCallback(lambda res: self.add_extra_node(self.numclients))
@ -149,11 +155,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
del test_connections
def test_upload_and_download_random_key(self):
self.basedir = "system/SystemTest/test_upload_and_download_random_key"
return self._test_upload_and_download(convergence=None)
def test_upload_and_download_convergent(self):
self.basedir = "system/SystemTest/test_upload_and_download_convergent"
return self._test_upload_and_download(convergence=b"some convergence string")
def _test_upload_and_download(self, convergence):
@ -473,9 +477,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def _corrupt_mutable_share(self, filename, which):
msf = MutableShareFile(filename)
datav = msf.readv([ (0, 1000000) ])
# Read more than share length:
datav = msf.readv([ (0, 10_000_000) ])
final_share = datav[0]
assert len(final_share) < 1000000 # ought to be truncated
assert len(final_share) < 10_000_000 # ought to be truncated
pieces = mutable_layout.unpack_share(final_share)
(seqnum, root_hash, IV, k, N, segsize, datalen,
verification_key, signature, share_hash_chain, block_hash_tree,
@ -515,13 +520,20 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
msf.writev( [(0, final_share)], None)
def test_mutable(self):
self.basedir = "system/SystemTest/test_mutable"
def test_mutable_sdmf(self):
"""SDMF mutables can be uploaded, downloaded, and many other things."""
return self._test_mutable(SDMF_VERSION)
def test_mutable_mdmf(self):
"""MDMF mutables can be uploaded, downloaded, and many other things."""
return self._test_mutable(MDMF_VERSION)
def _test_mutable(self, mutable_version):
DATA = b"initial contents go here." # 25 bytes % 3 != 0
DATA_uploadable = MutableData(DATA)
NEWDATA = b"new contents yay"
NEWDATA_uploadable = MutableData(NEWDATA)
NEWERDATA = b"this is getting old"
NEWERDATA = b"this is getting old" * 1_000_000
NEWERDATA_uploadable = MutableData(NEWERDATA)
d = self.set_up_nodes()
@ -529,7 +541,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def _create_mutable(res):
c = self.clients[0]
log.msg("starting create_mutable_file")
d1 = c.create_mutable_file(DATA_uploadable)
d1 = c.create_mutable_file(DATA_uploadable, mutable_version)
def _done(res):
log.msg("DONE: %s" % (res,))
self._mutable_node_1 = res
@ -551,27 +563,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
filename)
self.failUnlessEqual(rc, 0)
try:
share_type = 'SDMF' if mutable_version == SDMF_VERSION else 'MDMF'
self.failUnless("Mutable slot found:\n" in output)
self.failUnless("share_type: SDMF\n" in output)
self.assertIn(f"share_type: {share_type}\n", output)
peerid = idlib.nodeid_b2a(self.clients[client_num].nodeid)
self.failUnless(" WE for nodeid: %s\n" % peerid in output)
self.failUnless(" num_extra_leases: 0\n" in output)
self.failUnless(" secrets are for nodeid: %s\n" % peerid
in output)
self.failUnless(" SDMF contents:\n" in output)
self.failUnless(f" {share_type} contents:\n" in output)
self.failUnless(" seqnum: 1\n" in output)
self.failUnless(" required_shares: 3\n" in output)
self.failUnless(" total_shares: 10\n" in output)
self.failUnless(" segsize: 27\n" in output, (output, filename))
if mutable_version == SDMF_VERSION:
self.failUnless(" segsize: 27\n" in output, (output, filename))
self.failUnless(" datalen: 25\n" in output)
# the exact share_hash_chain nodes depends upon the sharenum,
# and is more of a hassle to compute than I want to deal with
# now
self.failUnless(" share_hash_chain: " in output)
self.failUnless(" block_hash_tree: 1 nodes\n" in output)
expected = (" verify-cap: URI:SSK-Verifier:%s:" %
str(base32.b2a(storage_index), "ascii"))
self.failUnless(expected in output)
if mutable_version == SDMF_VERSION:
expected = (" verify-cap: URI:SSK-Verifier:%s:" %
str(base32.b2a(storage_index), "ascii"))
else:
expected = (" verify-cap: URI:MDMF-Verifier:%s" %
str(base32.b2a(storage_index), "ascii"))
self.assertIn(expected, output)
except unittest.FailTest:
print()
print("dump-share output was:")
@ -691,7 +709,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# when we retrieve this, we should get three signature
# failures (where we've mangled seqnum, R, and segsize). The
# pubkey mangling
d.addCallback(_corrupt_shares)
if mutable_version == SDMF_VERSION:
# TODO Corrupting shares in test_systm doesn't work for MDMF right now
d.addCallback(_corrupt_shares)
d.addCallback(lambda res: self._newnode3.download_best_version())
d.addCallback(_check_download_5)
@ -699,7 +720,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def _check_empty_file(res):
# make sure we can create empty files, this usually screws up the
# segsize math
d1 = self.clients[2].create_mutable_file(MutableData(b""))
d1 = self.clients[2].create_mutable_file(MutableData(b""), mutable_version)
d1.addCallback(lambda newnode: newnode.download_best_version())
d1.addCallback(lambda res: self.failUnlessEqual(b"", res))
return d1
@ -746,7 +767,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# plaintext_hash check.
def test_filesystem(self):
self.basedir = "system/SystemTest/test_filesystem"
self.data = LARGE_DATA
d = self.set_up_nodes()
def _new_happy_semantics(ign):
@ -1713,7 +1733,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def test_filesystem_with_cli_in_subprocess(self):
# We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
self.basedir = "system/SystemTest/test_filesystem_with_cli_in_subprocess"
d = self.set_up_nodes()
def _new_happy_semantics(ign):
for c in self.clients:
@ -1794,9 +1813,21 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
class Connections(SystemTestMixin, unittest.TestCase):
FORCE_FOOLSCAP_FOR_STORAGE = True
def test_rref(self):
self.basedir = "system/Connections/rref"
# The way the listening port is created is via
# SameProcessStreamEndpointAssigner (allmydata.test.common), which then
# makes an endpoint string parsed by AdoptedServerPort. The latter does
# dup(fd), which results in the filedescriptor staying alive _until the
# test ends_. That means that when we disown the service, we still have
# the listening port there on the OS level! Just the resulting
# connections aren't handled. So this test relies on aggressive
# timeouts in the HTTP client and presumably some equivalent in
# Foolscap, since connection refused does _not_ happen.
self.basedir = "system/Connections/rref-foolscap-{}".format(
self.FORCE_FOOLSCAP_FOR_STORAGE
)
d = self.set_up_nodes(2)
def _start(ign):
self.c0 = self.clients[0]
@ -1812,9 +1843,13 @@ class Connections(SystemTestMixin, unittest.TestCase):
# now shut down the server
d.addCallback(lambda ign: self.clients[1].disownServiceParent())
# kill any persistent http connections that might continue to work
d.addCallback(lambda ign: self.close_idle_http_connections())
# and wait for the client to notice
def _poll():
return len(self.c0.storage_broker.get_connected_servers()) < 2
return len(self.c0.storage_broker.get_connected_servers()) == 1
d.addCallback(lambda ign: self.poll(_poll))
def _down(ign):
@ -1824,3 +1859,16 @@ class Connections(SystemTestMixin, unittest.TestCase):
self.assertEqual(storage_server, self.s1_storage_server)
d.addCallback(_down)
return d
class HTTPSystemTest(SystemTest):
"""HTTP storage protocol variant of the system tests."""
FORCE_FOOLSCAP_FOR_STORAGE = False
class HTTPConnections(Connections):
"""HTTP storage protocol variant of the connections tests."""
FORCE_FOOLSCAP_FOR_STORAGE = False

22
src/allmydata/test/test_testing.py
View File

@ -9,18 +9,7 @@
"""
Tests for the allmydata.testing helpers
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.internet.defer import (
inlineCallbacks,
@ -56,10 +45,12 @@ from testtools.matchers import (
IsInstance,
MatchesStructure,
AfterPreprocessing,
Contains,
)
from testtools.twistedsupport import (
succeeded,
)
from twisted.web.http import GONE
class FakeWebTest(SyncTestCase):
@ -144,7 +135,8 @@ class FakeWebTest(SyncTestCase):
def test_download_missing(self):
"""
Error if we download a capability that doesn't exist
The response to a request to download a capability that doesn't exist
is 410 (GONE).
"""
http_client = create_tahoe_treq_client()
@ -157,7 +149,11 @@ class FakeWebTest(SyncTestCase):
resp,
succeeded(
MatchesStructure(
code=Equals(500)
code=Equals(GONE),
content=AfterPreprocessing(
lambda m: m(),
succeeded(Contains(b"No data for")),
),
)
)
)

65
src/allmydata/test/web/test_web.py
View File

@ -1,19 +1,14 @@
"""
Ported to Python 3.
Tests for a bunch of web-related APIs.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_binary
import os.path, re, time
import treq
from urllib.parse import quote as urlquote, unquote as urlunquote
from base64 import urlsafe_b64encode
from bs4 import BeautifulSoup
@ -38,6 +33,7 @@ from allmydata.util import fileutil, base32, hashutil, jsonbytes as json
from allmydata.util.consumer import download_to_data
from allmydata.util.encodingutil import to_bytes
from ...util.connection_status import ConnectionStatus
from ...crypto.rsa import PublicKey, PrivateKey, create_signing_keypair, der_string_from_signing_key
from ..common import (
EMPTY_CLIENT_CONFIG,
FakeCHKFileNode,
@ -65,6 +61,7 @@ from allmydata.interfaces import (
MustBeReadonlyError,
)
from allmydata.mutable import servermap, publish, retrieve
from allmydata.mutable.common import derive_mutable_keys
from .. import common_util as testutil
from ..common_util import TimezoneMixin
from ..common_web import (
@ -93,6 +90,7 @@ class FakeNodeMaker(NodeMaker):
'happy': 7,
'max_segment_size':128*1024 # 1024=KiB
}
all_contents: dict[bytes, object]
def _create_lit(self, cap):
return FakeCHKFileNode(cap, self.all_contents)
def _create_immutable(self, cap):
@ -100,11 +98,19 @@ class FakeNodeMaker(NodeMaker):
def _create_mutable(self, cap):
return FakeMutableFileNode(None, None,
self.encoding_params, None,
self.all_contents).init_from_cap(cap)
def create_mutable_file(self, contents=b"", keysize=None,
version=SDMF_VERSION):
self.all_contents, None).init_from_cap(cap)
def create_mutable_file(self,
contents=None,
version=None,
keypair: tuple[PublicKey, PrivateKey] | None=None,
):
if contents is None:
contents = b""
if version is None:
version = SDMF_VERSION
n = FakeMutableFileNode(None, None, self.encoding_params, None,
self.all_contents)
self.all_contents, keypair)
return n.create(contents, version=version)
class FakeUploader(service.Service):
@ -2868,6 +2874,41 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
"Unknown format: foo",
method="post", data=body, headers=headers)
async def test_POST_upload_keypair(self) -> None:
"""
A *POST* creating a new mutable object may include a *private-key*
query argument giving a urlsafe-base64-encoded RSA private key to use
as the "signature key". The given signature key is used, rather than
a new one being generated.
"""
format = "sdmf"
priv, pub = create_signing_keypair(2048)
encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii")
filename = "predetermined-sdmf"
expected_content = self.NEWFILE_CONTENTS * 100
actual_cap = uri.from_string(await self.POST(
self.public_url +
f"/foo?t=upload&format={format}&private-key={encoded_privkey}",
file=(filename, expected_content),
))
# Ideally we would inspect the private ("signature") and public
# ("verification") keys but they are not made easily accessible here
# (ostensibly because we have a FakeMutableFileNode instead of a real
# one).
#
# So, instead, re-compute the writekey and fingerprint and compare
# those against the capability string.
expected_writekey, _, expected_fingerprint = derive_mutable_keys((pub, priv))
self.assertEqual(
(expected_writekey, expected_fingerprint),
(actual_cap.writekey, actual_cap.fingerprint),
)
# And the capability we got can be used to download the data we
# uploaded.
downloaded_content = await self.GET(f"/uri/{actual_cap.to_string().decode('ascii')}")
self.assertEqual(expected_content, downloaded_content)
def test_POST_upload_format(self):
def _check_upload(ign, format, uri_prefix, fn=None):
filename = format + ".txt"

10
src/allmydata/test/web/test_webish.py
View File

@ -202,6 +202,16 @@ class TahoeLAFSSiteTests(SyncTestCase):
),
)
def test_private_key_censoring(self):
"""
The log event for a request including a **private-key** query
argument has the private key value censored.
"""
self._test_censoring(
b"/uri?uri=URI:CHK:aaa:bbb&private-key=AAAAaaaabbbb==",
b"/uri?uri=[CENSORED]&private-key=[CENSORED]",
)
def test_uri_censoring(self):
"""
The log event for a request for **/uri/<CAP>** has the capability value

24
src/allmydata/testing/web.py
View File

@ -6,20 +6,12 @@
# This file is part of Tahoe-LAFS.
#
# See the docs/about.rst file for licensing information.
"""Test-helpers for clients that use the WebUI.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
Test-helpers for clients that use the WebUI.
"""
from __future__ import annotations
import hashlib
@ -54,6 +46,7 @@ import allmydata.uri
from allmydata.util import (
base32,
)
from ..util.dictutil import BytesKeyDict
__all__ = (
@ -147,7 +140,7 @@ class _FakeTahoeUriHandler(Resource, object):
isLeaf = True
data = attr.ib(default=attr.Factory(dict))
data: BytesKeyDict = attr.ib(default=attr.Factory(BytesKeyDict))
capability_generators = attr.ib(default=attr.Factory(dict))
def _generate_capability(self, kind):
@ -209,7 +202,7 @@ class _FakeTahoeUriHandler(Resource, object):
capability = None
for arg, value in uri.query:
if arg == u"uri":
capability = value
capability = value.encode("utf-8")
# it's legal to use the form "/uri/<capability>"
if capability is None and request.postpath and request.postpath[0]:
capability = request.postpath[0]
@ -221,10 +214,9 @@ class _FakeTahoeUriHandler(Resource, object):
# the user gave us a capability; if our Grid doesn't have any
# data for it, that's an error.
capability = capability.encode('ascii')
if capability not in self.data:
request.setResponseCode(http.BAD_REQUEST)
return u"No data for '{}'".format(capability.decode('ascii'))
request.setResponseCode(http.GONE)
return u"No data for '{}'".format(capability.decode('ascii')).encode("utf-8")
return self.data[capability]

34
src/allmydata/util/dictutil.py
View File

@ -1,21 +1,6 @@
"""
Tools to mess with dicts.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# IMPORTANT: We deliberately don't import dict. The issue is that we're
# subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of
# code that doesn't support it.
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
from six import ensure_str
class DictOfSets(dict):
def add(self, key, value):
@ -104,7 +89,7 @@ def _make_enforcing_override(K, method_name):
raise TypeError("{} must be of type {}".format(
repr(key), self.KEY_TYPE))
return getattr(dict, method_name)(self, key, *args, **kwargs)
f.__name__ = ensure_str(method_name)
f.__name__ = method_name
setattr(K, method_name, f)
for _method_name in ["__setitem__", "__getitem__", "setdefault", "get",
@ -113,18 +98,13 @@ for _method_name in ["__setitem__", "__getitem__", "setdefault", "get",
del _method_name
if PY2:
# No need for enforcement, can use either bytes or unicode as keys and it's
# fine.
BytesKeyDict = UnicodeKeyDict = dict
else:
class BytesKeyDict(_TypedKeyDict):
"""Keys should be bytes."""
class BytesKeyDict(_TypedKeyDict):
"""Keys should be bytes."""
KEY_TYPE = bytes
KEY_TYPE = bytes
class UnicodeKeyDict(_TypedKeyDict):
"""Keys should be unicode strings."""
class UnicodeKeyDict(_TypedKeyDict):
"""Keys should be unicode strings."""
KEY_TYPE = str
KEY_TYPE = str

149
src/allmydata/util/pipeline.py
View File

@ -1,149 +0,0 @@
"""
A pipeline of Deferreds.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.python import log
from allmydata.util.assertutil import precondition
class PipelineError(Exception):
"""One of the pipelined messages returned an error. The received Failure
object is stored in my .error attribute."""
def __init__(self, error):
self.error = error
def __repr__(self):
return "<PipelineError error=(%r)>" % (self.error,)
def __str__(self):
return "<PipelineError error=(%s)>" % (self.error,)
class SingleFileError(Exception):
"""You are not permitted to add a job to a full pipeline."""
class ExpandableDeferredList(defer.Deferred, object):
# like DeferredList(fireOnOneErrback=True) with a built-in
# gatherResults(), but you can add new Deferreds until you close it. This
# gives you a chance to add don't-complain-about-unhandled-error errbacks
# immediately after attachment, regardless of whether you actually end up
# wanting the list or not.
def __init__(self):
defer.Deferred.__init__(self)
self.resultsReceived = 0
self.resultList = []
self.failure = None
self.closed = False
def addDeferred(self, d):
precondition(not self.closed, "don't call addDeferred() on a closed ExpandableDeferredList")
index = len(self.resultList)
self.resultList.append(None)
d.addCallbacks(self._cbDeferred, self._ebDeferred,
callbackArgs=(index,))
return d
def close(self):
self.closed = True
self.checkForFinished()
def checkForFinished(self):
if not self.closed:
return
if self.called:
return
if self.failure:
self.errback(self.failure)
elif self.resultsReceived == len(self.resultList):
self.callback(self.resultList)
def _cbDeferred(self, res, index):
self.resultList[index] = res
self.resultsReceived += 1
self.checkForFinished()
return res
def _ebDeferred(self, f):
self.failure = f
self.checkForFinished()
return f
class Pipeline(object):
"""I manage a size-limited pipeline of Deferred operations, usually
callRemote() messages."""
def __init__(self, capacity):
self.capacity = capacity # how full we can be
self.gauge = 0 # how full we are
self.failure = None
self.waiting = [] # callers of add() who are blocked
self.unflushed = ExpandableDeferredList()
def add(self, _size, _func, *args, **kwargs):
# We promise that all the Deferreds we return will fire in the order
# they were returned. To make it easier to keep this promise, we
# prohibit multiple outstanding calls to add() .
if self.waiting:
raise SingleFileError
if self.failure:
return defer.fail(self.failure)
self.gauge += _size
fd = defer.maybeDeferred(_func, *args, **kwargs)
fd.addBoth(self._call_finished, _size)
self.unflushed.addDeferred(fd)
fd.addErrback(self._eat_pipeline_errors)
fd.addErrback(log.err, "_eat_pipeline_errors didn't eat it")
if self.gauge < self.capacity:
return defer.succeed(None)
d = defer.Deferred()
self.waiting.append(d)
return d
def flush(self):
if self.failure:
return defer.fail(self.failure)
d, self.unflushed = self.unflushed, ExpandableDeferredList()
d.close()
d.addErrback(self._flushed_error)
return d
def _flushed_error(self, f):
precondition(self.failure) # should have been set by _call_finished
return self.failure
def _call_finished(self, res, size):
self.gauge -= size
if isinstance(res, Failure):
res = Failure(PipelineError(res))
if not self.failure:
self.failure = res
if self.failure:
while self.waiting:
d = self.waiting.pop(0)
d.errback(self.failure)
else:
while self.waiting and (self.gauge < self.capacity):
d = self.waiting.pop(0)
d.callback(None)
# the d.callback() might trigger a new call to add(), which
# will raise our gauge and might cause the pipeline to be
# filled. So the while() loop gets a chance to tell the
# caller to stop.
return res
def _eat_pipeline_errors(self, f):
f.trap(PipelineError)
return None

Some files were not shown because too many files have changed in this diff Show More