ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-06-01 15:20:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 2916.grid-manager-integration-tests.2

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2023-07-03 10:55:33 -04:00
commit c4e6ea6379
132 changed files with 2955 additions and 1633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

247
.circleci/config.yml
View File

@ -11,22 +11,36 @@
# #
version: 2.1 version: 2.1
# A template that can be shared between the two different image-building # Every job that pushes a Docker image from Docker Hub must authenticate to
# it. Define a couple yaml anchors that can be used to supply the necessary
# credentials.
# First is a CircleCI job context which makes Docker Hub credentials available
# in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
dockerhub-context-template: &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
# Next is a Docker executor template that gets the credentials from the
# environment and supplies them to the executor.
dockerhub-auth-template: &DOCKERHUB_AUTH
- auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
# A template that can be shared between the two different image-building
# workflows. # workflows.
.images: &IMAGES .images: &IMAGES
jobs: jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide - "build-image-debian-11":
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub credentials
# available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-11": &DOCKERHUB_CONTEXT
<<: *DOCKERHUB_CONTEXT <<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04": - "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT <<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-22-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35": - "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT <<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8": - "build-image-oraclelinux-8":
@ -66,17 +80,30 @@ workflows:
- "ubuntu-20-04": - "ubuntu-20-04":
{} {}
- "ubuntu-22-04":
{}
# Equivalent to RHEL 8; CentOS 8 is dead. # Equivalent to RHEL 8; CentOS 8 is dead.
- "oraclelinux-8": - "oraclelinux-8":
{} {}
- "nixos": - "nixos":
name: "NixOS 22.11" name: "<<matrix.pythonVersion>>"
nixpkgs: "22.11" nixpkgs: "22.11"
matrix:
parameters:
pythonVersion:
- "python38"
- "python39"
- "python310"
- "nixos": - "nixos":
name: "NixOS unstable" name: "<<matrix.pythonVersion>>"
nixpkgs: "unstable" nixpkgs: "unstable"
matrix:
parameters:
pythonVersion:
- "python311"
# Eventually, test against PyPy 3.8 # Eventually, test against PyPy 3.8
#- "pypy27-buster": #- "pypy27-buster":
@ -113,30 +140,7 @@ workflows:
# Build as part of the workflow but only if requested. # Build as part of the workflow but only if requested.
when: "<< pipeline.parameters.build-images >>" when: "<< pipeline.parameters.build-images >>"
jobs: jobs:
dockerhub-auth-template:
# This isn't a real job. It doesn't get scheduled as part of any
# workflow. Instead, it's just a place we can hang a yaml anchor to
# finish the Docker Hub authentication configuration. Workflow jobs using
# the DOCKERHUB_CONTEXT anchor will have access to the environment
# variables used here. These variables will allow the Docker Hub image
# pull to be authenticated and hopefully avoid hitting and rate limits.
docker: &DOCKERHUB_AUTH
- image: "null"
auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
steps:
- run:
name: "CircleCI YAML schema conformity"
command: |
# This isn't a real command. We have to have something in this
# space, though, or the CircleCI yaml schema validator gets angry.
# Since this job is never scheduled this step is never run so the
# actual value here is irrelevant.
codechecks: codechecks:
docker: docker:
- <<: *DOCKERHUB_AUTH - <<: *DOCKERHUB_AUTH
@ -256,7 +260,7 @@ jobs:
name: "Submit coverage results" name: "Submit coverage results"
command: | command: |
if [ -n "${UPLOAD_COVERAGE}" ]; then if [ -n "${UPLOAD_COVERAGE}" ]; then
/tmp/venv/bin/codecov echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011"
fi fi
docker: docker:
@ -336,6 +340,16 @@ jobs:
<<: *UTF_8_ENVIRONMENT <<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py39" TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
ubuntu-22-04:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:22.04-py3.10"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py310"
oraclelinux-8: &RHEL_DERIV oraclelinux-8: &RHEL_DERIV
docker: docker:
- <<: *DOCKERHUB_AUTH - <<: *DOCKERHUB_AUTH
@ -374,56 +388,29 @@ jobs:
Reference the name of a niv-managed nixpkgs source (see `niv show` Reference the name of a niv-managed nixpkgs source (see `niv show`
and nix/sources.json) and nix/sources.json)
type: "string" type: "string"
pythonVersion:
description: >-
Reference the name of a Python package in nixpkgs to use.
type: "string"
docker: executor: "nix"
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.10.3"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
# allows us to push to CACHIX_NAME. We only need this set for
# `cachix use` in this step.
CACHIX_NAME: "tahoe-lafs-opensource"
steps: steps:
- "run": - "nix-build":
# Get cachix for Nix-friendly caching. nixpkgs: "<<parameters.nixpkgs>>"
name: "Install Basic Dependencies" pythonVersion: "<<parameters.pythonVersion>>"
command: | buildSteps:
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz" - "run":
nix-env \ name: "Unit Test"
--file $NIXPKGS \ command: |
--install \ # The dependencies are all built so we can allow more
-A cachix bash # parallelism here.
# Activate it for "binary substitution". This sets up source .circleci/lib.sh
# configuration tht lets Nix download something from the cache cache_if_able nix-build \
# instead of building it locally, if possible. --cores 8 \
cachix use "${CACHIX_NAME}" --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
- "checkout" nix/tests.nix
- "run":
# The Nix package doesn't know how to do this part, unfortunately.
name: "Generate version"
command: |
nix-shell \
-p 'python3.withPackages (ps: [ ps.setuptools ])' \
--run 'python setup.py update_version'
- "run":
name: "Test"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suites the build environment we're paying
# for (the free one!).
source .circleci/lib.sh
cache_if_able nix-build \
--cores 8 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
nix/tests.nix
typechecks: typechecks:
docker: docker:
@ -509,6 +496,15 @@ jobs:
PYTHON_VERSION: "3.9" PYTHON_VERSION: "3.9"
build-image-ubuntu-22-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "22.04"
PYTHON_VERSION: "3.10"
build-image-oraclelinux-8: build-image-oraclelinux-8:
<<: *BUILD_IMAGE <<: *BUILD_IMAGE
@ -527,7 +523,6 @@ jobs:
# build-image-pypy27-buster: # build-image-pypy27-buster:
# <<: *BUILD_IMAGE # <<: *BUILD_IMAGE
# environment: # environment:
# DISTRO: "pypy" # DISTRO: "pypy"
# TAG: "buster" # TAG: "buster"
@ -535,3 +530,87 @@ jobs:
# # setting up PyPy 3 in the image building toolchain. This value is just # # setting up PyPy 3 in the image building toolchain. This value is just
# # for constructing the right Docker image tag. # # for constructing the right Docker image tag.
# PYTHON_VERSION: "2" # PYTHON_VERSION: "2"
executors:
nix:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.10.3"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
# to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push
# to.
CACHIX_NAME: "tahoe-lafs-opensource"
commands:
nix-build:
parameters:
nixpkgs:
description: >-
Reference the name of a niv-managed nixpkgs source (see `niv show`
and nix/sources.json)
type: "string"
pythonVersion:
description: >-
Reference the name of a Python package in nixpkgs to use.
type: "string"
buildSteps:
description: >-
The build steps to execute after setting up the build environment.
type: "steps"
steps:
- "run":
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
nix-env \
--file $NIXPKGS \
--install \
-A cachix bash
# Activate it for "binary substitution". This sets up
# configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"
- "run":
# The Nix package doesn't know how to do this part, unfortunately.
name: "Generate version"
command: |
nix-shell \
-p 'python3.withPackages (ps: [ ps.setuptools ])' \
--run 'python setup.py update_version'
- "run":
name: "Build Dependencies"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suits the build environment we're paying
# for (the free one!).
source .circleci/lib.sh
# nix-shell will build all of the dependencies of the target but
# not the target itself.
cache_if_able nix-shell \
--run "" \
--cores 3 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
- "run":
name: "Build Package"
command: |
source .circleci/lib.sh
cache_if_able nix-build \
--cores 4 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
--argstr pythonVersion "<<parameters.pythonVersion>>" \
./default.nix
- steps: "<<parameters.buildSteps>>"

4
.circleci/create-virtualenv.sh
View File

@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
# above, it may still not be able to get us a compatible version unless we # above, it may still not be able to get us a compatible version unless we
# explicitly ask for one. # explicitly ask for one.
"${PIP}" install --upgrade setuptools==44.0.0 wheel "${PIP}" install --upgrade setuptools==44.0.0 wheel
# Just about every user of this image wants to use tox from the bootstrap
# virtualenv so go ahead and install it now.
"${PIP}" install "tox~=3.0"

26
.circleci/populate-wheelhouse.sh
View File

@ -3,18 +3,6 @@
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail set -euxo pipefail
# Basic Python packages that you just need to have around to do anything,
# practically speaking.
BASIC_DEPS="pip wheel"
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox~=3.0 codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
REPORTING_DEPS="python-subunit junitxml subunitreporter"
# The filesystem location of the wheelhouse which we'll populate with wheels # The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies. # for all of our dependencies.
WHEELHOUSE_PATH="$1" WHEELHOUSE_PATH="$1"
@ -41,15 +29,5 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
LANG="en_US.UTF-8" "${PIP}" \ LANG="en_US.UTF-8" "${PIP}" \
wheel \ wheel \
--wheel-dir "${WHEELHOUSE_PATH}" \ --wheel-dir "${WHEELHOUSE_PATH}" \
"${PROJECT_ROOT}"[test] \ "${PROJECT_ROOT}"[testenv] \
${BASIC_DEPS} \ "${PROJECT_ROOT}"[test]
${TEST_DEPS} \
${REPORTING_DEPS}
# Not strictly wheelhouse population but ... Note we omit basic deps here.
# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
# have to ask.
"${PIP}" \
install \
${TEST_DEPS} \
${REPORTING_DEPS}

6
.circleci/run-tests.sh
View File

@ -79,9 +79,10 @@ else
alternative="false" alternative="false"
fi fi
WORKDIR=/tmp/tahoe-lafs.tox
${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \
-c ${PROJECT_ROOT}/tox.ini \ -c ${PROJECT_ROOT}/tox.ini \
--workdir /tmp/tahoe-lafs.tox \ --workdir "${WORKDIR}" \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS} || "${alternative}" ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then
# Create a junitxml results area. # Create a junitxml results area.
mkdir -p "$(dirname "${JUNITXML}")" mkdir -p "$(dirname "${JUNITXML}")"
"${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
"${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
fi fi

7
.circleci/setup-virtualenv.sh
View File

@ -26,12 +26,7 @@ shift || :
# Tell pip where it can find any existing wheels. # Tell pip where it can find any existing wheels.
export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
export PIP_NO_INDEX="1"
# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
# between the time dependencies change and the images are re-built and (b) the
# upcoming-deprecations job wants to install some dependencies from github and
# it's awkward to get that done any earlier than the tox run. So, we don't
# set it.
# Get everything else installed in it, too. # Get everything else installed in it, too.
"${BOOTSTRAP_VENV}"/bin/tox \ "${BOOTSTRAP_VENV}"/bin/tox \

17
.github/workflows/ci.yml vendored
View File

@ -46,7 +46,6 @@ jobs:
matrix: matrix:
os: os:
- windows-latest - windows-latest
- ubuntu-latest
python-version: python-version:
- "3.8" - "3.8"
- "3.9" - "3.9"
@ -54,9 +53,9 @@ jobs:
- "3.11" - "3.11"
include: include:
# On macOS don't bother with 3.8, just to get faster builds. # On macOS don't bother with 3.8, just to get faster builds.
- os: macos-latest - os: macos-12
python-version: "3.9" python-version: "3.9"
- os: macos-latest - os: macos-12
python-version: "3.11" python-version: "3.11"
# We only support PyPy on Linux at the moment. # We only support PyPy on Linux at the moment.
- os: ubuntu-latest - os: ubuntu-latest
@ -80,7 +79,7 @@ jobs:
- name: Install Python packages - name: Install Python packages
run: | run: |
pip install --upgrade codecov "tox<4" tox-gh-actions setuptools pip install --upgrade "tox<4" tox-gh-actions setuptools
pip list pip list
- name: Display tool versions - name: Display tool versions
@ -166,16 +165,16 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- os: macos-latest - os: macos-12
python-version: "3.9" python-version: "3.11"
force-foolscap: false force-foolscap: false
- os: windows-latest - os: windows-latest
python-version: "3.9" python-version: "3.11"
force-foolscap: false force-foolscap: false
# 22.04 has some issue with Tor at the moment: # 22.04 has some issue with Tor at the moment:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
- os: ubuntu-20.04 - os: ubuntu-20.04
python-version: "3.11" python-version: "3.10"
force-foolscap: false force-foolscap: false
steps: steps:
@ -249,7 +248,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
os: os:
- macos-10.15 - macos-12
- windows-latest - windows-latest
- ubuntu-latest - ubuntu-latest
python-version: python-version:

2
.gitignore vendored
View File

@ -53,3 +53,5 @@ zope.interface-*.egg
# This is the plaintext of the private environment needed for some CircleCI # This is the plaintext of the private environment needed for some CircleCI
# operations. It's never supposed to be checked in. # operations. It's never supposed to be checked in.
secret-env-plain secret-env-plain
.ruff_cache

5
.readthedocs.yaml
View File

@ -1,5 +1,10 @@
version: 2 version: 2
build:
os: ubuntu-22.04
tools:
python: "3.10"
python: python:
install: install:
- requirements: docs/requirements.txt - requirements: docs/requirements.txt

18
.ruff.toml Normal file
View File

@ -0,0 +1,18 @@
select = [
# Pyflakes checks
"F",
# Prohibit tabs:
"W191",
# No trailing whitespace:
"W291",
"W293",
# Make sure we bind closure variables in a loop (equivalent to pylint
# cell-var-from-loop):
"B023",
# Don't silence exceptions in finally by accident:
"B012",
# Don't use mutable default arguments:
"B006",
# Errors from PyLint:
"PLE",
]

6
default.nix
View File

@ -32,11 +32,7 @@ in
}: }:
with (pkgs.${pythonVersion}.override { with (pkgs.${pythonVersion}.override {
packageOverrides = self: super: { packageOverrides = import ./nix/python-overrides.nix;
# Some dependencies aren't packaged in nixpkgs so supply our own packages.
pycddl = self.callPackage ./nix/pycddl.nix { };
txi2p = self.callPackage ./nix/txi2p.nix { };
};
}).pkgs; }).pkgs;
callPackage ./nix/tahoe-lafs.nix { callPackage ./nix/tahoe-lafs.nix {
# Select whichever package extras were requested. # Select whichever package extras were requested.

5
docs/performance.rst
View File

@ -82,8 +82,9 @@ network: A
memory footprint: N/K*A memory footprint: N/K*A
notes: Tahoe-LAFS generates a new RSA keypair for each mutable file that it notes:
publishes to a grid. This takes up to 1 or 2 seconds on a typical desktop PC. Tahoe-LAFS generates a new RSA keypair for each mutable file that it publishes to a grid.
This takes around 100 milliseconds on a relatively high-end laptop from 2021.
Part of the process of encrypting, encoding, and uploading a mutable file to a Part of the process of encrypting, encoding, and uploading a mutable file to a
Tahoe-LAFS grid requires that the entire file be in memory at once. For larger Tahoe-LAFS grid requires that the entire file be in memory at once. For larger

429
docs/proposed/http-storage-node-protocol.rst
View File

@ -3,7 +3,7 @@
Storage Node Protocol ("Great Black Swamp", "GBS") Storage Node Protocol ("Great Black Swamp", "GBS")
================================================== ==================================================
The target audience for this document is Tahoe-LAFS developers. The target audience for this document is developers working on Tahoe-LAFS or on an alternate implementation intended to be interoperable.
After reading this document, After reading this document,
one should expect to understand how Tahoe-LAFS clients interact over the network with Tahoe-LAFS storage nodes. one should expect to understand how Tahoe-LAFS clients interact over the network with Tahoe-LAFS storage nodes.
@ -64,6 +64,10 @@ Glossary
lease renew secret lease renew secret
a short secret string which storage servers required to be presented before allowing a particular lease to be renewed a short secret string which storage servers required to be presented before allowing a particular lease to be renewed
The key words
"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL"
in this document are to be interpreted as described in RFC 2119.
Motivation Motivation
---------- ----------
@ -119,8 +123,8 @@ An HTTP-based protocol can make use of TLS in largely the same way to provide th
Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation
(rather than the standard "web" rules for validation). (rather than the standard "web" rules for validation).
Requirements Design Requirements
------------ -------------------
Security Security
~~~~~~~~ ~~~~~~~~
@ -189,6 +193,9 @@ Solutions
An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described below. An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described below.
This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol. This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol.
Summary (Non-normative)
~~~~~~~~~~~~~~~~~~~~~~~
Communication with the storage node will take place using TLS. Communication with the storage node will take place using TLS.
The TLS version and configuration will be dictated by an ongoing understanding of best practices. The TLS version and configuration will be dictated by an ongoing understanding of best practices.
The storage node will present an x509 certificate during the TLS handshake. The storage node will present an x509 certificate during the TLS handshake.
@ -237,10 +244,10 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the *
.. note:: .. note::
Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate). Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate).
They are encoded with Base32 for a length of 32 bytes. They are encoded with `Base32`_ for a length of 32 bytes.
SPKI information discussed here is 32 bytes (SHA256 digest). SPKI information discussed here is 32 bytes (SHA256 digest).
They would be encoded in Base32 for a length of 52 bytes. They would be encoded in `Base32`_ for a length of 52 bytes.
`base64url`_ provides a more compact encoding of the information while remaining URL-compatible. `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
This would encode the SPKI information for a length of merely 43 bytes. This would encode the SPKI information for a length of merely 43 bytes.
SHA1, SHA1,
the current Foolscap hash function, the current Foolscap hash function,
@ -329,15 +336,117 @@ and shares.
A particular resource is addressed by the HTTP request path. A particular resource is addressed by the HTTP request path.
Details about the interface are encoded in the HTTP message body. Details about the interface are encoded in the HTTP message body.
String Encoding
~~~~~~~~~~~~~~~
.. _Base32:
Base32
!!!!!!
Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6.
That is, the alphabet is:
.. list-table:: Base32 Alphabet
:header-rows: 1
* - Value
- Encoding
- Value
- Encoding
- Value
- Encoding
- Value
- Encoding
* - 0
- a
- 9
- j
- 18
- s
- 27
- 3
* - 1
- b
- 10
- k
- 19
- t
- 28
- 4
* - 2
- c
- 11
- l
- 20
- u
- 29
- 5
* - 3
- d
- 12
- m
- 21
- v
- 30
- 6
* - 4
- e
- 13
- n
- 22
- w
- 31
- 7
* - 5
- f
- 14
- o
- 23
- x
-
-
* - 6
- g
- 15
- p
- 24
- y
-
-
* - 7
- h
- 16
- q
- 25
- z
-
-
* - 8
- i
- 17
- r
- 26
- 2
-
-
Message Encoding Message Encoding
~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~
The preferred encoding for HTTP message bodies is `CBOR`_. Clients and servers MUST use the ``Content-Type`` and ``Accept`` header fields as specified in `RFC 9110`_ for message body negotiation.
A request may be submitted using an alternate encoding by declaring this in the ``Content-Type`` header.
A request may indicate its preference for an alternate encoding in the response using the ``Accept`` header.
These two headers are used in the typical way for an HTTP application.
The only other encoding support for which is currently recommended is JSON. The encoding for HTTP message bodies SHOULD be `CBOR`_.
Clients submitting requests using this encoding MUST include a ``Content-Type: application/cbor`` request header field.
A request MAY be submitted using an alternate encoding by declaring this in the ``Content-Type`` header field.
A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field.
A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field.
Clients and servers MAY support additional request and response message body encodings.
Clients and servers SHOULD support ``application/json`` request and response message body encoding.
For HTTP messages carrying binary share data, For HTTP messages carrying binary share data,
this is expected to be a particularly poor encoding. this is expected to be a particularly poor encoding.
However, However,
@ -350,10 +459,23 @@ Because of the simple types used throughout
and the equivalence described in `RFC 7049`_ and the equivalence described in `RFC 7049`_
these examples should be representative regardless of which of these two encodings is chosen. these examples should be representative regardless of which of these two encodings is chosen.
The one exception is sets. There are two exceptions to this rule.
For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details. 1. Sets
Sets will be represented as JSON lists in examples because JSON doesn't support sets. !!!!!!!
For CBOR messages,
any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
Tag 6.258 is used to indicate sets in CBOR;
see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details.
The JSON encoding does not support sets.
Sets MUST be represented as arrays in JSON-encoded messages.
2. Bytes
!!!!!!!!
The CBOR encoding natively supports a bytes type while the JSON encoding does not.
Bytes MUST be represented as strings giving the `Base64`_ representation of the original bytes value.
HTTP Design HTTP Design
~~~~~~~~~~~ ~~~~~~~~~~~
@ -368,29 +490,50 @@ one branch contains all of the share data;
another branch contains all of the lease data; another branch contains all of the lease data;
etc. etc.
An ``Authorization`` header in requests is required for all endpoints. Clients and servers MUST use the ``Authorization`` header field,
The standard HTTP authorization protocol is used. as specified in `RFC 9110`_,
The authentication *type* used is ``Tahoe-LAFS``. for authorization of all requests to all endpoints specified here.
The swissnum from the NURL used to locate the storage service is used as the *credentials*. The authentication *type* MUST be ``Tahoe-LAFS``.
If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response. Clients MUST present the `Base64`_-encoded representation of the swissnum from the NURL used to locate the storage service as the *credentials*.
There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers. If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
If these are:
Requests to certain endpoints MUST include additional secrets in the ``X-Tahoe-Authorization`` headers field.
The endpoints which require these secrets are:
* ``PUT /storage/v1/lease/:storage_index``:
The secrets included MUST be ``lease-renew-secret`` and ``lease-cancel-secret``.
* ``POST /storage/v1/immutable/:storage_index``:
The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``upload-secret``.
* ``PATCH /storage/v1/immutable/:storage_index/:share_number``:
The secrets included MUST be ``upload-secret``.
* ``PUT /storage/v1/immutable/:storage_index/:share_number/abort``:
The secrets included MUST be ``upload-secret``.
* ``POST /storage/v1/mutable/:storage_index/read-test-write``:
The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``write-enabler``.
If these secrets are:
1. Missing. 1. Missing.
2. The wrong length. 2. The wrong length.
3. Not the expected kind of secret. 3. Not the expected kind of secret.
4. They are otherwise unparseable before they are actually semantically used. 4. They are otherwise unparseable before they are actually semantically used.
the server will respond with ``400 BAD REQUEST``. the server MUST respond with ``400 BAD REQUEST`` and perform no other processing of the message.
401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug. 401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug.
If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent. If authorization using the secret fails,
then the server MUST send a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
Encoding Encoding
~~~~~~~~ ~~~~~~~~
* ``storage_index`` should be base32 encoded (RFC3548) in URLs. * ``storage_index`` MUST be `Base32`_ encoded in URLs.
* ``share_number`` MUST be a decimal representation
General General
~~~~~~~ ~~~~~~~
@ -398,21 +541,27 @@ General
``GET /storage/v1/version`` ``GET /storage/v1/version``
!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve information about the version of the storage server. This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service.
Information is returned as an encoded mapping. The response MUST validate against this CDDL schema::
For example::
{'http://allmydata.org/tahoe/protocols/storage/v1' => {
'maximum-immutable-share-size' => uint
'maximum-mutable-share-size' => uint
'available-space' => uint
}
'application-version' => bstr
}
The server SHOULD populate as many fields as possible with accurate information about its behavior.
For fields which relate to a specific API
the semantics are documented below in the section for that API.
For fields that are more general than a single API the semantics are as follows:
* available-space:
The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests.
The value is a number of bytes.
{ "http://allmydata.org/tahoe/protocols/storage/v1" :
{ "maximum-immutable-share-size": 1234,
"maximum-mutable-share-size": 1235,
"available-space": 123456,
"tolerates-immutable-read-overrun": true,
"delete-mutable-shares-with-zero-length-writev": true,
"fills-holes-with-zero-bytes": true,
"prevents-read-past-end-of-share-data": true
},
"application-version": "1.13.0"
}
``PUT /storage/v1/lease/:storage_index`` ``PUT /storage/v1/lease/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -471,21 +620,37 @@ Writing
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Initialize an immutable storage index with some buckets. Initialize an immutable storage index with some buckets.
The buckets may have share data written to them once. The server MUST allow share data to be written to the buckets at most one time.
A lease is also created for the shares. The server MAY create a lease for the buckets.
Details of the buckets to create are encoded in the request body. Details of the buckets to create are encoded in the request body.
The request body MUST validate against this CDDL schema::
{
share-numbers: #6.258([0*256 uint])
allocated-size: uint
}
For example:: For example::
{"share-numbers": [1, 7, ...], "allocated-size": 12345} {"share-numbers": [1, 7, ...], "allocated-size": 12345}
The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations. The server SHOULD accept a value for **allocated-size** that is less than or equal to the lesser of the values of the server's version message's **maximum-immutable-share-size** or **available-space** values.
The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
For example:: For example::
X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret> X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret> X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
X-Tahoe-Authorization: upload-secret <base64-upload-secret> X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response body includes encoded information about the created buckets. The response body MUST include encoded information about the created buckets.
The response body MUST validate against this CDDL schema::
{
already-have: #6.258([0*256 uint])
allocated: #6.258([0*256 uint])
}
For example:: For example::
{"already-have": [1, ...], "allocated": [7, ...]} {"already-have": [1, ...], "allocated": [7, ...]}
@ -542,27 +707,35 @@ Rejected designs for upload secrets:
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Write data for the indicated share. Write data for the indicated share.
The share number must belong to the storage index. The share number MUST belong to the storage index.
The request body is the raw share data (i.e., ``application/octet-stream``). The request body MUST be the raw share data (i.e., ``application/octet-stream``).
*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed. The request MUST include a *Content-Range* header field;
for large transfers this allows partially complete uploads to be resumed.
For example, For example,
a 1MiB share can be divided in to eight separate 128KiB chunks. a 1MiB share can be divided in to eight separate 128KiB chunks.
Each chunk can be uploaded in a separate request. Each chunk can be uploaded in a separate request.
Each request can include a *Content-Range* value indicating its placement within the complete share. Each request can include a *Content-Range* value indicating its placement within the complete share.
If any one of these requests fails then at most 128KiB of upload work needs to be retried. If any one of these requests fails then at most 128KiB of upload work needs to be retried.
The server must recognize when all of the data has been received and mark the share as complete The server MUST recognize when all of the data has been received and mark the share as complete
(which it can do because it was informed of the size when the storage index was initialized). (which it can do because it was informed of the size when the storage index was initialized).
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret> X-Tahoe-Authorization: upload-secret <base64-upload-secret>
Responses: Responses:
* When a chunk that does not complete the share is successfully uploaded the response is ``OK``. * When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``.
The response body indicates the range of share data that has yet to be uploaded. The response body MUST indicate the range of share data that has yet to be uploaded.
That is:: The response body MUST validate against this CDDL schema::
{
required: [0* {begin: uint, end: uint}]
}
For example::
{ "required": { "required":
[ { "begin": <byte position, inclusive> [ { "begin": <byte position, inclusive>
@ -573,11 +746,12 @@ Responses:
] ]
} }
* When the chunk that completes the share is successfully uploaded the response is ``CREATED``. * When the chunk that completes the share is successfully uploaded the response MUST be ``CREATED``.
* If the *Content-Range* for a request covers part of the share that has already, * If the *Content-Range* for a request covers part of the share that has already,
and the data does not match already written data, and the data does not match already written data,
the response is ``CONFLICT``. the response MUST be ``CONFLICT``.
At this point the only thing to do is abort the upload and start from scratch (see below). In this case the client MUST abort the upload.
The client MAY then restart the upload from scratch.
Discussion Discussion
`````````` ``````````
@ -603,34 +777,42 @@ From RFC 7231::
This cancels an *in-progress* upload. This cancels an *in-progress* upload.
The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret:: The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::
X-Tahoe-Authorization: upload-secret <base64-upload-secret> X-Tahoe-Authorization: upload-secret <base64-upload-secret>
The response code: If there is an incomplete upload with a matching upload-secret then the server MUST consider the abort to have succeeded.
In this case the response MUST be ``OK``.
* When the upload is still in progress and therefore the abort has succeeded, The server MUST respond to all future requests as if the operations related to this upload did not take place.
the response is ``OK``.
Future uploads can start from scratch with no pre-existing upload state stored on the server.
* If the uploaded has already finished, the response is 405 (Method Not Allowed)
and no change is made.
If there is no incomplete upload with a matching upload-secret then the server MUST respond with ``Method Not Allowed`` (405).
The server MUST make no client-visible changes to its state in this case.
``POST /storage/v1/immutable/:storage_index/:share_number/corrupt`` ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Advise the server the data read from the indicated share was corrupt. The Advise the server the data read from the indicated share was corrupt.
request body includes an human-meaningful text string with details about the The request body includes an human-meaningful text string with details about the corruption.
corruption. It also includes potentially important details about the share. It also includes potentially important details about the share.
The request body MUST validate against this CDDL schema::
{
reason: tstr .size (1..32765)
}
For example:: For example::
{"reason": "expected hash abcd, got hash efgh"} {"reason": "expected hash abcd, got hash efgh"}
.. share-type, storage-index, and share-number are inferred from the URL The report pertains to the immutable share with a **storage index** and **share number** given in the request path.
If the identified **storage index** and **share number** are known to the server then the response SHOULD be accepted and made available to server administrators.
In this case the response SHOULD be ``OK``.
If the response is not accepted then the response SHOULD be ``Not Found`` (404).
The response code is OK (200) by default, or NOT FOUND (404) if the share Discussion
couldn't be found. ``````````
The seemingly odd length limit on ``reason`` is chosen so that the *encoded* representation of the message is limited to 32768.
Reading Reading
~~~~~~~ ~~~~~~~
@ -638,26 +820,36 @@ Reading
``GET /storage/v1/immutable/:storage_index/shares`` ``GET /storage/v1/immutable/:storage_index/shares``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve a list (semantically, a set) indicating all shares available for the Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index.
indicated storage index. For example:: The response body MUST validate against this CDDL schema::
#6.258([0*256 uint])
For example::
[1, 5] [1, 5]
An unknown storage index results in an empty list. If the **storage index** in the request path is not known to the server then the response MUST include an empty list.
``GET /storage/v1/immutable/:storage_index/:share_number`` ``GET /storage/v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Read a contiguous sequence of bytes from one share in one bucket. Read a contiguous sequence of bytes from one share in one bucket.
The response body is the raw share data (i.e., ``application/octet-stream``). The response body MUST be the raw share data (i.e., ``application/octet-stream``).
The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). The ``Range`` header MAY be used to request exactly one ``bytes`` range,
Interpretation and response behavior is as specified in RFC 7233 § 4.1. in which case the response code MUST be ``Partial Content`` (206).
Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. Interpretation and response behavior MUST be as specified in RFC 7233 § 4.1.
Multiple ranges in a single request are *not* supported;
open-ended ranges are also not supported.
Clients MUST NOT send requests using these features.
If the response reads beyond the end of the data, the response may be shorter than the requested range. If the response reads beyond the end of the data,
The resulting ``Content-Range`` header will be consistent with the returned data. the response MUST be shorter than the requested range.
It MUST contain all data up to the end of the share and then end.
The resulting ``Content-Range`` header MUST be consistent with the returned data.
If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. If the response to a query is an empty range,
the server MUST send a ``No Content`` (204) response.
Discussion Discussion
`````````` ``````````
@ -696,13 +888,27 @@ The first write operation on a mutable storage index creates it
(that is, (that is,
there is no separate "create this storage index" operation as there is for the immutable storage index type). there is no separate "create this storage index" operation as there is for the immutable storage index type).
The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets:: The request MUST include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets::
X-Tahoe-Authorization: write-enabler <base64-write-enabler-secret> X-Tahoe-Authorization: write-enabler <base64-write-enabler-secret>
X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret> X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret> X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
The request body includes test, read, and write vectors for the operation. The request body MUST include test, read, and write vectors for the operation.
The request body MUST validate against this CDDL schema::
{
"test-write-vectors": {
0*256 share_number : {
"test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}]
"write": [* {"offset": uint, "data": bstr}]
"new-length": uint / null
}
}
"read-vector": [0*30 {"offset": uint, "size": uint}]
}
share_number = uint
For example:: For example::
{ {
@ -725,6 +931,14 @@ For example::
The response body contains a boolean indicating whether the tests all succeed The response body contains a boolean indicating whether the tests all succeed
(and writes were applied) and a mapping giving read data (pre-write). (and writes were applied) and a mapping giving read data (pre-write).
The response body MUST validate against this CDDL schema::
{
"success": bool,
"data": {0*256 share_number: [0* bstr]}
}
share_number = uint
For example:: For example::
{ {
@ -736,8 +950,17 @@ For example::
} }
} }
A test vector or read vector that read beyond the boundaries of existing data will return nothing for any bytes past the end. A client MAY send a test vector or read vector to bytes beyond the end of existing data.
As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length. In this case a server MUST behave as if the test or read vector referred to exactly as much data exists.
For example,
consider the case where the server has 5 bytes of data for a particular share.
If a client sends a read vector with an ``offset`` of 1 and a ``size`` of 4 then the server MUST respond with all of the data except the first byte.
If a client sends a read vector with the same ``offset`` and a ``size`` of 5 (or any larger value) then the server MUST respond in the same way.
Similarly,
if there is no data at all,
an empty byte string is returned no matter what the offset or length.
Reading Reading
~~~~~~~ ~~~~~~~
@ -746,23 +969,34 @@ Reading
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Retrieve a set indicating all shares available for the indicated storage index. Retrieve a set indicating all shares available for the indicated storage index.
For example (this is shown as list, since it will be list for JSON, but will be set for CBOR):: The response body MUST validate against this CDDL schema::
#6.258([0*256 uint])
For example::
[1, 5] [1, 5]
``GET /storage/v1/mutable/:storage_index/:share_number`` ``GET /storage/v1/mutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index`` Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``.
The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content). The response body MUST be the raw share data (i.e., ``application/octet-stream``).
Interpretation and response behavior is as specified in RFC 7233 § 4.1. The ``Range`` header MAY be used to request exactly one ``bytes`` range,
Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported. in which case the response code MUST be ``Partial Content`` (206).
Interpretation and response behavior MUST be specified in RFC 7233 § 4.1.
Multiple ranges in a single request are *not* supported;
open-ended ranges are also not supported.
Clients MUST NOT send requests using these features.
If the response reads beyond the end of the data, the response may be shorter than the requested range. If the response reads beyond the end of the data,
The resulting ``Content-Range`` header will be consistent with the returned data. the response MUST be shorter than the requested range.
It MUST contain all data up to the end of the share and then end.
The resulting ``Content-Range`` header MUST be consistent with the returned data.
If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used. If the response to a query is an empty range,
the server MUST send a ``No Content`` (204) response.
``POST /storage/v1/mutable/:storage_index/:share_number/corrupt`` ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt``
@ -774,6 +1008,9 @@ Just like the immutable version.
Sample Interactions Sample Interactions
------------------- -------------------
This section contains examples of client/server interactions to help illuminate the above specification.
This section is non-normative.
Immutable Data Immutable Data
~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~
@ -926,10 +1163,16 @@ otherwise it will read a byte which won't match `b""`::
204 NO CONTENT 204 NO CONTENT
.. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4
.. _RFC 4648: https://tools.ietf.org/html/rfc4648
.. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4
.. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4
.. _RFC 9110: https://tools.ietf.org/html/rfc9110
.. _CBOR: http://cbor.io/ .. _CBOR: http://cbor.io/
.. [#] .. [#]
@ -974,7 +1217,7 @@ otherwise it will read a byte which won't match `b""`::
spki_encoded = urlsafe_b64encode(spki_sha256) spki_encoded = urlsafe_b64encode(spki_sha256)
assert spki_encoded == tub_id assert spki_encoded == tub_id
Note we use `base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32. Note we use `unpadded base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.
.. [#] .. [#]
https://www.cvedetails.com/cve/CVE-2017-5638/ https://www.cvedetails.com/cve/CVE-2017-5638/
@ -985,6 +1228,6 @@ otherwise it will read a byte which won't match `b""`::
.. [#] .. [#]
https://efail.de/ https://efail.de/
.. _base64url: https://tools.ietf.org/html/rfc7515#appendix-C .. _unpadded base64url: https://tools.ietf.org/html/rfc7515#appendix-C
.. _attacking SHA1: https://en.wikipedia.org/wiki/SHA-1#Attacks .. _attacking SHA1: https://en.wikipedia.org/wiki/SHA-1#Attacks

10
docs/specifications/dirnodes.rst
View File

@ -267,7 +267,7 @@ How well does this design meet the goals?
value, so there are no opportunities for staleness value, so there are no opportunities for staleness
9. monotonicity: VERY: the single point of access also protects against 9. monotonicity: VERY: the single point of access also protects against
retrograde motion retrograde motion
Confidentiality leaks in the storage servers Confidentiality leaks in the storage servers
@ -332,8 +332,9 @@ MDMF design rules allow for efficient random-access reads from the middle of
the file, which would give the index something useful to point at. the file, which would give the index something useful to point at.
The current SDMF design generates a new RSA public/private keypair for each The current SDMF design generates a new RSA public/private keypair for each
directory. This takes considerable time and CPU effort, generally one or two directory. This takes some time and CPU effort (around 100 milliseconds on a
seconds per directory. We have designed (but not yet built) a DSA-based relatively high-end 2021 laptop) per directory.
We have designed (but not yet built) a DSA-based
mutable file scheme which will use shared parameters to reduce the mutable file scheme which will use shared parameters to reduce the
directory-creation effort to a bare minimum (picking a random number instead directory-creation effort to a bare minimum (picking a random number instead
of generating two random primes). of generating two random primes).
@ -363,7 +364,7 @@ single child, looking up a single child) would require pulling or pushing a
lot of unrelated data, increasing network overhead (and necessitating lot of unrelated data, increasing network overhead (and necessitating
test-and-set semantics for the modification side, which increases the chances test-and-set semantics for the modification side, which increases the chances
that a user operation will fail, making it more challenging to provide that a user operation will fail, making it more challenging to provide
promises of atomicity to the user). promises of atomicity to the user).
It would also make it much more difficult to enable the delegation It would also make it much more difficult to enable the delegation
("sharing") of specific directories. Since each aggregate "realm" provides ("sharing") of specific directories. Since each aggregate "realm" provides
@ -469,4 +470,3 @@ Preventing delegation between communication parties is just as pointless as
asking Bob to forget previously accessed files. However, there may be value asking Bob to forget previously accessed files. However, there may be value
to configuring the UI to ask Carol to not share files with Bob, or to to configuring the UI to ask Carol to not share files with Bob, or to
removing all files from Bob's view at the same time his access is revoked. removing all files from Bob's view at the same time his access is revoked.

186
integration/conftest.py
View File

@ -1,6 +1,10 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import annotations
import os
import sys import sys
import shutil import shutil
from time import sleep from time import sleep
@ -14,6 +18,7 @@ from eliot import (
log_call, log_call,
) )
from twisted.python.filepath import FilePath
from twisted.python.procutils import which from twisted.python.procutils import which
from twisted.internet.defer import DeferredList from twisted.internet.defer import DeferredList
from twisted.internet.error import ( from twisted.internet.error import (
@ -41,7 +46,16 @@ from .grid import (
create_flog_gatherer, create_flog_gatherer,
create_grid, create_grid,
) )
from allmydata.node import read_config
# No reason for HTTP requests to take longer than four minutes in the
# integration tests. See allmydata/scripts/common_http.py for usage.
os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "240"
# Make Foolscap logging go into Twisted logging, so that integration test logs
# include extra information
# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst):
os.environ["FLOGTOTWISTED"] = "1"
# pytest customization hooks # pytest customization hooks
@ -108,7 +122,7 @@ def port_allocator(reactor):
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
@log_call(action_type=u"integration:temp_dir", include_args=[]) @log_call(action_type=u"integration:temp_dir", include_args=[])
def temp_dir(request): def temp_dir(request) -> str:
""" """
Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir
""" """
@ -166,22 +180,16 @@ def introducer_furl(introducer, temp_dir):
return introducer.furl return introducer.furl
@pytest.fixture(scope='session') @pytest.fixture
@log_call( @log_call(
action_type=u"integration:tor:introducer", action_type=u"integration:tor:introducer",
include_args=["temp_dir", "flog_gatherer"], include_args=["temp_dir", "flog_gatherer"],
include_result=False, include_result=False,
) )
def tor_introducer(reactor, temp_dir, flog_gatherer, request): def tor_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_tor
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_tor') intro_dir = join(temp_dir, 'introducer_tor')
print("making introducer", intro_dir) print("making Tor introducer in {}".format(intro_dir))
print("(this can take tens of seconds to allocate Onion address)")
if not exists(intro_dir): if not exists(intro_dir):
mkdir(intro_dir) mkdir(intro_dir)
@ -192,20 +200,25 @@ log_gatherer.furl = {log_furl}
request, request,
( (
'create-introducer', 'create-introducer',
'--tor-control-port', 'tcp:localhost:8010', # The control port should agree with the configuration of the
# Tor network we bootstrap with chutney.
'--tor-control-port', 'tcp:localhost:8007',
'--hide-ip',
'--listen=tor', '--listen=tor',
intro_dir, intro_dir,
), ),
) )
pytest_twisted.blockon(done_proto.done) pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff # adjust a few settings
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: config = read_config(intro_dir, "tub.port")
f.write(config) config.set_config("node", "nickname", "introducer-tor")
config.set_config("node", "web.port", "4561")
config.set_config("node", "log_gatherer.furl", flog_gatherer)
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command. # "start" command.
protocol = _MagicTextProtocol('introducer running') protocol = _MagicTextProtocol('introducer running', "tor_introducer")
transport = _tahoe_runner_optional_coverage( transport = _tahoe_runner_optional_coverage(
protocol, protocol,
reactor, reactor,
@ -224,17 +237,20 @@ log_gatherer.furl = {log_furl}
pass pass
request.addfinalizer(cleanup) request.addfinalizer(cleanup)
print("Waiting for introducer to be ready...")
pytest_twisted.blockon(protocol.magic_seen) pytest_twisted.blockon(protocol.magic_seen)
print("Introducer ready.")
return transport return transport
@pytest.fixture(scope='session') @pytest.fixture
def tor_introducer_furl(tor_introducer, temp_dir): def tor_introducer_furl(tor_introducer, temp_dir):
furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl') furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl')
while not exists(furl_fname): while not exists(furl_fname):
print("Don't see {} yet".format(furl_fname)) print("Don't see {} yet".format(furl_fname))
sleep(.1) sleep(.1)
furl = open(furl_fname, 'r').read() furl = open(furl_fname, 'r').read()
print(f"Found Tor introducer furl: {furl} in {furl_fname}")
return furl return furl
@ -278,12 +294,9 @@ def alice(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
web_port="tcp:9980:interface=localhost", web_port="tcp:9980:interface=localhost",
storage=False, storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
) )
) )
await_client_ready(process) pytest_twisted.blockon(await_client_ready(process))
# 1. Create a new RW directory cap: # 1. Create a new RW directory cap:
cli(process, "create-alias", "test") cli(process, "create-alias", "test")
@ -314,7 +327,7 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}
# 4. Restart the node with new SFTP config. # 4. Restart the node with new SFTP config.
pytest_twisted.blockon(process.restart_async(reactor, request)) pytest_twisted.blockon(process.restart_async(reactor, request))
await_client_ready(process) pytest_twisted.blockon(await_client_ready(process))
print(f"Alice pid: {process.transport.pid}") print(f"Alice pid: {process.transport.pid}")
return process return process
@ -329,22 +342,37 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques
storage=False, storage=False,
) )
) )
await_client_ready(process) pytest_twisted.blockon(await_client_ready(process))
return process return process
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
@pytest.mark.skipif(sys.platform.startswith('win'), @pytest.mark.skipif(sys.platform.startswith('win'),
'Tor tests are unstable on Windows') 'Tor tests are unstable on Windows')
def chutney(reactor, temp_dir): def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
# Try to find Chutney already installed in the environment.
try:
import chutney
except ImportError:
# Nope, we'll get our own in a moment.
pass
else:
# We already have one, just use it.
return (
# from `checkout/lib/chutney/__init__.py` we want to get back to
# `checkout` because that's the parent of the directory with all
# of the network definitions. So, great-grand-parent.
FilePath(chutney.__file__).parent().parent().parent().path,
# There's nothing to add to the environment.
{},
)
chutney_dir = join(temp_dir, 'chutney') chutney_dir = join(temp_dir, 'chutney')
mkdir(chutney_dir) mkdir(chutney_dir)
# TODO: missing = [exe for exe in ["tor", "tor-gencert"] if not which(exe)]
if missing:
# check for 'tor' binary explicitly and emit a "skip" if we can't pytest.skip(f"Some command-line tools not found: {missing}")
# find it
# XXX yuck! should add a setup.py to chutney so we can at least # XXX yuck! should add a setup.py to chutney so we can at least
# "pip install <path to tarball>" and/or depend on chutney in "pip # "pip install <path to tarball>" and/or depend on chutney in "pip
@ -357,7 +385,7 @@ def chutney(reactor, temp_dir):
'git', 'git',
( (
'git', 'clone', 'git', 'clone',
'https://git.torproject.org/chutney.git', 'https://gitlab.torproject.org/tpo/core/chutney.git',
chutney_dir, chutney_dir,
), ),
env=environ, env=environ,
@ -373,94 +401,68 @@ def chutney(reactor, temp_dir):
( (
'git', '-C', chutney_dir, 'git', '-C', chutney_dir,
'reset', '--hard', 'reset', '--hard',
'c825cba0bcd813c644c6ac069deeb7347d3200ee' 'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2'
), ),
env=environ, env=environ,
) )
pytest_twisted.blockon(proto.done) pytest_twisted.blockon(proto.done)
return chutney_dir return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
@pytest.fixture(scope='session') @pytest.fixture(scope='session')
@pytest.mark.skipif(sys.platform.startswith('win'), @pytest.mark.skipif(sys.platform.startswith('win'),
reason='Tor tests are unstable on Windows') reason='Tor tests are unstable on Windows')
def tor_network(reactor, temp_dir, chutney, request): def tor_network(reactor, temp_dir, chutney, request):
"""
Build a basic Tor network.
# this is the actual "chutney" script at the root of a chutney checkout :param chutney: The root directory of a Chutney checkout and a dict of
chutney_dir = chutney additional environment variables to set so a Python process can use
chut = join(chutney_dir, 'chutney') it.
# now, as per Chutney's README, we have to create the network :return: None
# ./chutney configure networks/basic """
# ./chutney start networks/basic chutney_root, chutney_env = chutney
basic_network = join(chutney_root, 'networks', 'basic')
env = environ.copy() env = environ.copy()
env.update({"PYTHONPATH": join(chutney_dir, "lib")}) env.update(chutney_env)
proto = _DumpOutputProtocol(None) env.update({
reactor.spawnProcess( # default is 60, probably too short for reliable automated use.
proto, "CHUTNEY_START_TIME": "600",
sys.executable, })
( chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
sys.executable, '-m', 'chutney.TorNet', 'configure', def chutney(argv):
join(chutney_dir, 'networks', 'basic'), proto = _DumpOutputProtocol(None)
),
path=join(chutney_dir),
env=env,
)
pytest_twisted.blockon(proto.done)
proto = _DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'start',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env=env,
)
pytest_twisted.blockon(proto.done)
# print some useful stuff
proto = _CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'chutney.TorNet', 'status',
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env=env,
)
try:
pytest_twisted.blockon(proto.done)
except ProcessTerminated:
print("Chutney.TorNet status failed (continuing):")
print(proto.output.getvalue())
def cleanup():
print("Tearing down Chutney Tor network")
proto = _CollectOutputProtocol()
reactor.spawnProcess( reactor.spawnProcess(
proto, proto,
sys.executable, sys.executable,
( chutney_argv + argv,
sys.executable, '-m', 'chutney.TorNet', 'stop', path=join(chutney_root),
join(chutney_dir, 'networks', 'basic'),
),
path=join(chutney_dir),
env=env, env=env,
) )
return proto.done
# now, as per Chutney's README, we have to create the network
pytest_twisted.blockon(chutney(("configure", basic_network)))
# before we start the network, ensure we will tear down at the end
def cleanup():
print("Tearing down Chutney Tor network")
try: try:
block_with_timeout(proto.done, reactor) block_with_timeout(chutney(("stop", basic_network)), reactor)
except ProcessTerminated: except ProcessTerminated:
# If this doesn't exit cleanly, that's fine, that shouldn't fail # If this doesn't exit cleanly, that's fine, that shouldn't fail
# the test suite. # the test suite.
pass pass
request.addfinalizer(cleanup) request.addfinalizer(cleanup)
return chut pytest_twisted.blockon(chutney(("start", basic_network)))
pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network)))
# print some useful stuff
try:
pytest_twisted.blockon(chutney(("status", basic_network)))
except ProcessTerminated:
print("Chutney.TorNet status failed (continuing)")

4
integration/grid.py
View File

@ -99,7 +99,7 @@ def create_flog_gatherer(reactor, request, temp_dir, flog_binary):
) )
yield out_protocol.done yield out_protocol.done
twistd_protocol = _MagicTextProtocol("Gatherer waiting at") twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer")
twistd_process = reactor.spawnProcess( twistd_process = reactor.spawnProcess(
twistd_protocol, twistd_protocol,
which('twistd')[0], which('twistd')[0],
@ -341,7 +341,7 @@ def create_introducer(reactor, request, temp_dir, flog_gatherer, port):
# on windows, "tahoe start" means: run forever in the foreground, # on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent # but on linux it means daemonize. "tahoe run" is consistent
# between platforms. # between platforms.
protocol = _MagicTextProtocol('introducer running') protocol = _MagicTextProtocol('introducer running', "introducer")
transport = _tahoe_runner_optional_coverage( transport = _tahoe_runner_optional_coverage(
protocol, protocol,
reactor, reactor,

39
integration/test_get_put.py
View File

@ -4,11 +4,11 @@ and stdout.
""" """
from subprocess import Popen, PIPE, check_output, check_call from subprocess import Popen, PIPE, check_output, check_call
import sys
import pytest import pytest
from pytest_twisted import ensureDeferred
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.threads import blockingCallFromThread
from twisted.internet.defer import Deferred
from .util import run_in_thread, cli, reconfigure from .util import run_in_thread, cli, reconfigure
@ -50,6 +50,7 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir):
assert read_bytes(tempfile) == DATA assert read_bytes(tempfile) == DATA
@run_in_thread
def test_get_to_stdout(alice, get_put_alias, tmpdir): def test_get_to_stdout(alice, get_put_alias, tmpdir):
""" """
It's possible to upload a file, and then download it to stdout. It's possible to upload a file, and then download it to stdout.
@ -67,6 +68,7 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir):
assert p.wait() == 0 assert p.wait() == 0
@run_in_thread
def test_large_file(alice, get_put_alias, tmp_path): def test_large_file(alice, get_put_alias, tmp_path):
""" """
It's possible to upload and download a larger file. It's possible to upload and download a larger file.
@ -85,12 +87,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
assert outfile.read_bytes() == tempfile.read_bytes() assert outfile.read_bytes() == tempfile.read_bytes()
@pytest.mark.skipif( @run_in_thread
sys.platform.startswith("win"), def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
reason="reconfigure() has issues on Windows"
)
@ensureDeferred
async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
""" """
Tahoe-LAFS used to have a default max segment size of 128KB, and is now Tahoe-LAFS used to have a default max segment size of 128KB, and is now
1MB. Test that an upload created when 128KB was the default can be 1MB. Test that an upload created when 128KB was the default can be
@ -103,22 +101,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
with tempfile.open("wb") as f: with tempfile.open("wb") as f:
f.write(large_data) f.write(large_data)
async def set_segment_size(segment_size): def set_segment_size(segment_size):
await reconfigure( return blockingCallFromThread(
reactor, reactor,
request, lambda: Deferred.fromCoroutine(reconfigure(
alice, reactor,
(1, 1, 1), request,
None, alice,
max_segment_size=segment_size (1, 1, 1),
) None,
max_segment_size=segment_size
))
)
# 1. Upload file 1 with default segment size set to 1MB # 1. Upload file 1 with default segment size set to 1MB
await set_segment_size(1024 * 1024) set_segment_size(1024 * 1024)
cli(alice, "put", str(tempfile), "getput:seg1024kb") cli(alice, "put", str(tempfile), "getput:seg1024kb")
# 2. Download file 1 with default segment size set to 128KB # 2. Download file 1 with default segment size set to 128KB
await set_segment_size(128 * 1024) set_segment_size(128 * 1024)
assert large_data == check_output( assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"] ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
) )
@ -127,7 +128,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
cli(alice, "put", str(tempfile), "getput:seg128kb") cli(alice, "put", str(tempfile), "getput:seg128kb")
# 4. Download file 2 with default segment size set to 1MB # 4. Download file 2 with default segment size set to 1MB
await set_segment_size(1024 * 1024) set_segment_size(1024 * 1024)
assert large_data == check_output( assert large_data == check_output(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"] ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
) )

49
integration/test_i2p.py
View File

@ -2,26 +2,11 @@
Integration tests for I2P support. Integration tests for I2P support.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys import sys
from os.path import join, exists from os.path import join, exists
from os import mkdir from os import mkdir, environ
from time import sleep from time import sleep
from shutil import which
if PY2:
def which(path):
# This will result in skipping I2P tests on Python 2. Oh well.
return None
else:
from shutil import which
from eliot import log_call from eliot import log_call
@ -38,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready
from allmydata.test.common import ( from allmydata.test.common import (
write_introducer, write_introducer,
) )
from allmydata.node import read_config
if which("docker") is None: if which("docker") is None:
pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)
@ -50,7 +37,7 @@ if sys.platform.startswith('win'):
@pytest.fixture @pytest.fixture
def i2p_network(reactor, temp_dir, request): def i2p_network(reactor, temp_dir, request):
"""Fixture to start up local i2pd.""" """Fixture to start up local i2pd."""
proto = util._MagicTextProtocol("ephemeral keys") proto = util._MagicTextProtocol("ephemeral keys", "i2pd")
reactor.spawnProcess( reactor.spawnProcess(
proto, proto,
which("docker"), which("docker"),
@ -62,6 +49,7 @@ def i2p_network(reactor, temp_dir, request):
"--log=stdout", "--log=stdout",
"--loglevel=info" "--loglevel=info"
), ),
env=environ,
) )
def cleanup(): def cleanup():
@ -82,13 +70,6 @@ def i2p_network(reactor, temp_dir, request):
include_result=False, include_result=False,
) )
def i2p_introducer(reactor, temp_dir, flog_gatherer, request): def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_i2p
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_i2p') intro_dir = join(temp_dir, 'introducer_i2p')
print("making introducer", intro_dir) print("making introducer", intro_dir)
@ -108,12 +89,14 @@ log_gatherer.furl = {log_furl}
pytest_twisted.blockon(done_proto.done) pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff # over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: config = read_config(intro_dir, "tub.port")
f.write(config) config.set_config("node", "nickname", "introducer_i2p")
config.set_config("node", "web.port", "4563")
config.set_config("node", "log_gatherer.furl", flog_gatherer)
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command. # "start" command.
protocol = util._MagicTextProtocol('introducer running') protocol = util._MagicTextProtocol('introducer running', "introducer")
transport = util._tahoe_runner_optional_coverage( transport = util._tahoe_runner_optional_coverage(
protocol, protocol,
reactor, reactor,
@ -147,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
@pytest_twisted.inlineCallbacks @pytest_twisted.inlineCallbacks
@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons")
def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
@ -170,7 +154,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
sys.executable, '-b', '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'carol_i2p'), '-d', join(temp_dir, 'carol_i2p'),
'put', gold_path, 'put', gold_path,
) ),
env=environ,
) )
yield proto.done yield proto.done
cap = proto.output.getvalue().strip().split()[-1] cap = proto.output.getvalue().strip().split()[-1]
@ -184,7 +169,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
sys.executable, '-b', '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'dave_i2p'), '-d', join(temp_dir, 'dave_i2p'),
'get', cap, 'get', cap,
) ),
env=environ,
) )
yield proto.done yield proto.done
@ -211,7 +197,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
'--hide-ip', '--hide-ip',
'--listen', 'i2p', '--listen', 'i2p',
node_dir.path, node_dir.path,
) ),
env=environ,
) )
yield proto.done yield proto.done

14
integration/test_servers_of_happiness.py
View File

@ -1,17 +1,10 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys import sys
from os.path import join from os.path import join
from os import environ
from . import util from . import util
@ -29,7 +22,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
happy=7, happy=7,
total=10, total=10,
) )
util.await_client_ready(edna) yield util.await_client_ready(edna)
node_dir = join(temp_dir, 'edna') node_dir = join(temp_dir, 'edna')
@ -43,7 +36,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
sys.executable, '-b', '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', node_dir, '-d', node_dir,
'put', __file__, 'put', __file__,
] ],
env=environ,
) )
try: try:
yield proto.done yield proto.done

130
integration/test_tor.py
View File

@ -1,17 +1,10 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys import sys
from os.path import join from os.path import join
from os import environ
import pytest import pytest
import pytest_twisted import pytest_twisted
@ -25,6 +18,8 @@ from twisted.python.filepath import (
from allmydata.test.common import ( from allmydata.test.common import (
write_introducer, write_introducer,
) )
from allmydata.client import read_config
from allmydata.util.deferredutil import async_to_deferred
# see "conftest.py" for the fixtures (e.g. "tor_network") # see "conftest.py" for the fixtures (e.g. "tor_network")
@ -35,18 +30,28 @@ from allmydata.test.common import (
if sys.platform.startswith('win'): if sys.platform.startswith('win'):
pytest.skip('Skipping Tor tests on Windows', allow_module_level=True) pytest.skip('Skipping Tor tests on Windows', allow_module_level=True)
if PY2:
pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True)
@pytest_twisted.inlineCallbacks @pytest_twisted.inlineCallbacks
def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) """
dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) Two nodes and an introducer all configured to use Tahoe.
util.await_client_ready(carol, minimum_number_of_servers=2)
util.await_client_ready(dave, minimum_number_of_servers=2) The two nodes can talk to the introducer and each other: we upload to one
node, read from the other.
"""
carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave)
@async_to_deferred
async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: util.TahoeProcess, download_from: util.TahoeProcess):
"""
Ensure both nodes are connected to "a grid" by uploading something via one
node, and retrieve it using the other.
"""
# ensure both nodes are connected to "a grid" by uploading
# something via carol, and retrieve it using dave.
gold_path = join(temp_dir, "gold") gold_path = join(temp_dir, "gold")
with open(gold_path, "w") as f: with open(gold_path, "w") as f:
f.write( f.write(
@ -63,13 +68,14 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
sys.executable, sys.executable,
( (
sys.executable, '-b', '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'carol'), '-d', upload_to.node_dir,
'put', gold_path, 'put', gold_path,
) ),
env=environ,
) )
yield proto.done await proto.done
cap = proto.output.getvalue().strip().split()[-1] cap = proto.output.getvalue().strip().split()[-1]
print("TEH CAP!", cap) print("capability: {}".format(cap))
proto = util._CollectOutputProtocol(capture_stderr=False) proto = util._CollectOutputProtocol(capture_stderr=False)
reactor.spawnProcess( reactor.spawnProcess(
@ -77,77 +83,83 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
sys.executable, sys.executable,
( (
sys.executable, '-b', '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'dave'), '-d', download_from.node_dir,
'get', cap, 'get', cap,
) ),
env=environ,
) )
yield proto.done await proto.done
download_got = proto.output.getvalue().strip()
dave_got = proto.output.getvalue().strip() assert download_got == open(gold_path, 'rb').read().strip()
assert dave_got == open(gold_path, 'rb').read().strip()
@pytest_twisted.inlineCallbacks @pytest_twisted.inlineCallbacks
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl): def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess:
node_dir = FilePath(temp_dir).child(name) node_dir = FilePath(temp_dir).child(name)
web_port = "tcp:{}:interface=localhost".format(control_port + 2000) web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
if node_dir.exists(): if node_dir.exists():
raise RuntimeError( raise RuntimeError(
"A node already exists in '{}'".format(node_dir) "A node already exists in '{}'".format(node_dir)
) )
print("creating", node_dir) print(f"creating {node_dir.path} with introducer {introducer_furl}")
node_dir.makedirs() node_dir.makedirs()
proto = util._DumpOutputProtocol(None) proto = util._DumpOutputProtocol(None)
reactor.spawnProcess( reactor.spawnProcess(
proto, proto,
sys.executable, sys.executable,
( (
sys.executable, '-m', 'allmydata.scripts.runner', sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'create-node', 'create-node',
'--nickname', name, '--nickname', name,
'--webport', web_port,
'--introducer', introducer_furl, '--introducer', introducer_furl,
'--hide-ip', '--hide-ip',
'--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
'--listen', 'tor', '--listen', 'tor',
'--shares-needed', '1',
'--shares-happy', '1',
'--shares-total', str(shares_total),
node_dir.path, node_dir.path,
),
env=environ,
) )
)
yield proto.done yield proto.done
# Which services should this client connect to? # Which services should this client connect to?
write_introducer(node_dir, "default", introducer_furl) write_introducer(node_dir, "default", introducer_furl)
with node_dir.child('tahoe.cfg').open('w') as f: util.basic_node_configuration(request, flog_gatherer, node_dir.path)
node_config = '''
[node]
nickname = %(name)s
web.port = %(web_port)s
web.static = public_html
log_gatherer.furl = %(log_furl)s
[tor] config = read_config(node_dir.path, "tub.port")
control.port = tcp:localhost:%(control_port)d config.set_config("tor", "onion", "true")
onion.external_port = 3457 config.set_config("tor", "onion.external_port", "3457")
onion.local_port = %(local_port)d config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
onion = true config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
onion.private_key_file = private/tor_onion.privkey
[client]
shares.needed = 1
shares.happy = 1
shares.total = 2
''' % {
'name': name,
'web_port': web_port,
'log_furl': flog_gatherer,
'control_port': control_port,
'local_port': control_port + 1000,
}
node_config = node_config.encode("utf-8")
f.write(node_config)
print("running") print("running")
result = yield util._run_node(reactor, node_dir.path, request, None) result = yield util._run_node(reactor, node_dir.path, request, None)
print("okay, launched") print("okay, launched")
return result return result
@pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS')
@pytest_twisted.inlineCallbacks
def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
"""
A normal node (normie) and a normal introducer are configured, and one node
(anonymoose) which is configured to be anonymous by talking via Tor.
Anonymoose should be able to communicate with normie.
TODO how to ensure that anonymoose is actually using Tor?
"""
normie = yield util._create_node(
reactor, request, temp_dir, introducer_furl, flog_gatherer, "normie",
web_port="tcp:9989:interface=localhost",
storage=True, needed=1, happy=1, total=1,
)
yield util.await_client_ready(normie)
anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1)
yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600)
yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose)

22
integration/test_web.py
View File

@ -14,17 +14,21 @@ from __future__ import annotations
import time import time
from urllib.parse import unquote as url_unquote, quote as url_quote from urllib.parse import unquote as url_unquote, quote as url_quote
from twisted.internet.threads import deferToThread
import allmydata.uri import allmydata.uri
from allmydata.util import jsonbytes as json from allmydata.util import jsonbytes as json
from . import util from . import util
from .util import run_in_thread
import requests import requests
import html5lib import html5lib
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from pytest_twisted import ensureDeferred import pytest_twisted
@run_in_thread
def test_index(alice): def test_index(alice):
""" """
we can download the index file we can download the index file
@ -32,6 +36,7 @@ def test_index(alice):
util.web_get(alice, u"") util.web_get(alice, u"")
@run_in_thread
def test_index_json(alice): def test_index_json(alice):
""" """
we can download the index file as json we can download the index file as json
@ -41,6 +46,7 @@ def test_index_json(alice):
json.loads(data) json.loads(data)
@run_in_thread
def test_upload_download(alice): def test_upload_download(alice):
""" """
upload a file, then download it via readcap upload a file, then download it via readcap
@ -70,6 +76,7 @@ def test_upload_download(alice):
assert str(data, "utf-8") == FILE_CONTENTS assert str(data, "utf-8") == FILE_CONTENTS
@run_in_thread
def test_put(alice): def test_put(alice):
""" """
use PUT to create a file use PUT to create a file
@ -89,6 +96,7 @@ def test_put(alice):
assert cap.needed_shares == int(cfg.get_config("client", "shares.needed")) assert cap.needed_shares == int(cfg.get_config("client", "shares.needed"))
@run_in_thread
def test_helper_status(storage_nodes): def test_helper_status(storage_nodes):
""" """
successfully GET the /helper_status page successfully GET the /helper_status page
@ -101,6 +109,7 @@ def test_helper_status(storage_nodes):
assert str(dom.h1.string) == u"Helper Status" assert str(dom.h1.string) == u"Helper Status"
@run_in_thread
def test_deep_stats(alice): def test_deep_stats(alice):
""" """
create a directory, do deep-stats on it and prove the /operations/ create a directory, do deep-stats on it and prove the /operations/
@ -178,7 +187,7 @@ def test_deep_stats(alice):
time.sleep(.5) time.sleep(.5)
@util.run_in_thread @run_in_thread
def test_status(alice): def test_status(alice):
""" """
confirm we get something sensible from /status and the various sub-types confirm we get something sensible from /status and the various sub-types
@ -244,7 +253,7 @@ def test_status(alice):
assert found_download, "Failed to find the file we downloaded in the status-page" assert found_download, "Failed to find the file we downloaded in the status-page"
@ensureDeferred @pytest_twisted.ensureDeferred
async def test_directory_deep_check(reactor, request, alice): async def test_directory_deep_check(reactor, request, alice):
""" """
use deep-check and confirm the result pages work use deep-check and confirm the result pages work
@ -256,7 +265,10 @@ async def test_directory_deep_check(reactor, request, alice):
total = 4 total = 4
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None) await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
await deferToThread(_test_directory_deep_check_blocking, alice)
def _test_directory_deep_check_blocking(alice):
# create a directory # create a directory
resp = requests.post( resp = requests.post(
util.node_url(alice.node_dir, u"uri"), util.node_url(alice.node_dir, u"uri"),
@ -417,6 +429,7 @@ async def test_directory_deep_check(reactor, request, alice):
assert dom is not None, "Operation never completed" assert dom is not None, "Operation never completed"
@run_in_thread
def test_storage_info(storage_nodes): def test_storage_info(storage_nodes):
""" """
retrieve and confirm /storage URI for one storage node retrieve and confirm /storage URI for one storage node
@ -428,6 +441,7 @@ def test_storage_info(storage_nodes):
) )
@run_in_thread
def test_storage_info_json(storage_nodes): def test_storage_info_json(storage_nodes):
""" """
retrieve and confirm /storage?t=json URI for one storage node retrieve and confirm /storage?t=json URI for one storage node
@ -442,6 +456,7 @@ def test_storage_info_json(storage_nodes):
assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000 assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000
@run_in_thread
def test_introducer_info(introducer): def test_introducer_info(introducer):
""" """
retrieve and confirm /introducer URI for the introducer retrieve and confirm /introducer URI for the introducer
@ -460,6 +475,7 @@ def test_introducer_info(introducer):
assert "subscription_summary" in data assert "subscription_summary" in data
@run_in_thread
def test_mkdir_with_children(alice): def test_mkdir_with_children(alice):
""" """
create a directory using ?t=mkdir-with-children create a directory using ?t=mkdir-with-children

150
integration/util.py
View File

@ -12,7 +12,7 @@ import sys
import time import time
import json import json
from os import mkdir, environ from os import mkdir, environ
from os.path import exists, join from os.path import exists, join, basename
from io import StringIO, BytesIO from io import StringIO, BytesIO
from subprocess import check_output from subprocess import check_output
@ -117,7 +117,6 @@ class _CollectOutputProtocol(ProcessProtocol):
self.output.write(data) self.output.write(data)
def errReceived(self, data): def errReceived(self, data):
print("ERR: {!r}".format(data))
if self.capture_stderr: if self.capture_stderr:
self.output.write(data) self.output.write(data)
@ -153,8 +152,9 @@ class _MagicTextProtocol(ProcessProtocol):
and then .callback()s on self.done and .errback's if the process exits and then .callback()s on self.done and .errback's if the process exits
""" """
def __init__(self, magic_text): def __init__(self, magic_text: str, name: str) -> None:
self.magic_seen = Deferred() self.magic_seen = Deferred()
self.name = f"{name}: "
self.exited = Deferred() self.exited = Deferred()
self._magic_text = magic_text self._magic_text = magic_text
self._output = StringIO() self._output = StringIO()
@ -164,7 +164,8 @@ class _MagicTextProtocol(ProcessProtocol):
def outReceived(self, data): def outReceived(self, data):
data = str(data, sys.stdout.encoding) data = str(data, sys.stdout.encoding)
sys.stdout.write(data) for line in data.splitlines():
sys.stdout.write(self.name + line + "\n")
self._output.write(data) self._output.write(data)
if not self.magic_seen.called and self._magic_text in self._output.getvalue(): if not self.magic_seen.called and self._magic_text in self._output.getvalue():
print("Saw '{}' in the logs".format(self._magic_text)) print("Saw '{}' in the logs".format(self._magic_text))
@ -172,7 +173,8 @@ class _MagicTextProtocol(ProcessProtocol):
def errReceived(self, data): def errReceived(self, data):
data = str(data, sys.stderr.encoding) data = str(data, sys.stderr.encoding)
sys.stdout.write(data) for line in data.splitlines():
sys.stdout.write(self.name + line + "\n")
def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@ -320,7 +322,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
""" """
if magic_text is None: if magic_text is None:
magic_text = "client running" magic_text = "client running"
protocol = _MagicTextProtocol(magic_text) protocol = _MagicTextProtocol(magic_text, basename(node_dir))
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command. # "start" command.
@ -349,6 +351,36 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
return d return d
def basic_node_configuration(request, flog_gatherer, node_dir: str):
"""
Setup common configuration options for a node, given a ``pytest`` request
fixture.
"""
config_path = join(node_dir, 'tahoe.cfg')
config = get_config(config_path)
set_config(
config,
u'node',
u'log_gatherer.furl',
flog_gatherer,
)
force_foolscap = request.config.getoption("force_foolscap")
assert force_foolscap in (True, False)
set_config(
config,
'storage',
'force_foolscap',
str(force_foolscap),
)
set_config(
config,
'client',
'force_foolscap',
str(force_foolscap),
)
write_config(FilePath(config_path), config)
def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port, def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
storage=True, storage=True,
magic_text=None, magic_text=None,
@ -389,29 +421,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
created_d = done_proto.done created_d = done_proto.done
def created(_): def created(_):
config_path = join(node_dir, 'tahoe.cfg') basic_node_configuration(request, flog_gatherer.furl, node_dir)
config = get_config(config_path)
set_config(
config,
u'node',
u'log_gatherer.furl',
flog_gatherer.furl,
)
force_foolscap = request.config.getoption("force_foolscap")
assert force_foolscap in (True, False)
set_config(
config,
'storage',
'force_foolscap',
str(force_foolscap),
)
set_config(
config,
'client',
'force_foolscap',
str(force_foolscap),
)
write_config(FilePath(config_path), config)
created_d.addCallback(created) created_d.addCallback(created)
d = Deferred() d = Deferred()
@ -468,6 +478,31 @@ class FileShouldVanishException(Exception):
) )
def run_in_thread(f):
"""Decorator for integration tests that runs code in a thread.
Because we're using pytest_twisted, tests that rely on the reactor are
expected to return a Deferred and use async APIs so the reactor can run.
In the case of the integration test suite, it launches nodes in the
background using Twisted APIs. The nodes stdout and stderr is read via
Twisted code. If the reactor doesn't run, reads don't happen, and
eventually the buffers fill up, and the nodes block when they try to flush
logs.
We can switch to Twisted APIs (treq instead of requests etc.), but
sometimes it's easier or expedient to just have a blocking test. So this
decorator allows you to run the test in a thread, and the reactor can keep
running in the main thread.
See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
"""
@wraps(f)
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test
def await_file_contents(path, contents, timeout=15, error_if=None): def await_file_contents(path, contents, timeout=15, error_if=None):
""" """
wait up to `timeout` seconds for the file at `path` (any path-like wait up to `timeout` seconds for the file at `path` (any path-like
@ -593,6 +628,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
return resp.content return resp.content
@run_in_thread
def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
""" """
Uses the status API to wait for a client-type node (in `tahoe`, a Uses the status API to wait for a client-type node (in `tahoe`, a
@ -617,24 +653,25 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
print("waiting because '{}'".format(e)) print("waiting because '{}'".format(e))
time.sleep(1) time.sleep(1)
continue continue
servers = js['servers']
if len(js['servers']) < minimum_number_of_servers: if len(servers) < minimum_number_of_servers:
print(f"waiting because insufficient servers (expected at least {minimum_number_of_servers})") print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})")
time.sleep(1) time.sleep(1)
continue continue
print(
f"Now: {time.ctime()}\n"
f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}"
)
server_times = [ server_times = [
server['last_received_data'] server['last_received_data']
for server in js['servers'] for server in servers
] ]
# if any times are null/None that server has never been # check that all times are 'recent enough' (it's OK if _some_ servers
# contacted (so it's down still, probably) # are down, we just want to make sure a sufficient number are up)
if any(t is None for t in server_times): if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers:
print("waiting because at least one server not contacted")
time.sleep(1)
continue
# check that all times are 'recent enough'
if any([time.time() - t > liveness for t in server_times]):
print("waiting because at least one server too old") print("waiting because at least one server too old")
time.sleep(1) time.sleep(1)
continue continue
@ -660,30 +697,6 @@ def generate_ssh_key(path):
f.write(s.encode("ascii")) f.write(s.encode("ascii"))
def run_in_thread(f):
"""Decorator for integration tests that runs code in a thread.
Because we're using pytest_twisted, tests that rely on the reactor are
expected to return a Deferred and use async APIs so the reactor can run.
In the case of the integration test suite, it launches nodes in the
background using Twisted APIs. The nodes stdout and stderr is read via
Twisted code. If the reactor doesn't run, reads don't happen, and
eventually the buffers fill up, and the nodes block when they try to flush
logs.
We can switch to Twisted APIs (treq instead of requests etc.), but
sometimes it's easier or expedient to just have a blocking test. So this
decorator allows you to run the test in a thread, and the reactor can keep
running in the main thread.
See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
"""
@wraps(f)
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test
@frozen @frozen
class CHK: class CHK:
""" """
@ -830,16 +843,11 @@ async def reconfigure(reactor, request, node: TahoeProcess,
) )
if changed: if changed:
# TODO reconfigure() seems to have issues on Windows. If you need to
# use it there, delete this assert and try to figure out what's going
# on...
assert not sys.platform.startswith("win")
# restart the node # restart the node
print(f"Restarting {node.node_dir} for ZFEC reconfiguration") print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
await node.restart_async(reactor, request) await node.restart_async(reactor, request)
print("Restarted. Waiting for ready state.") print("Restarted. Waiting for ready state.")
await_client_ready(node) await await_client_ready(node)
print("Ready.") print("Ready.")
else: else:
print("Config unchanged, not restarting.") print("Config unchanged, not restarting.")

17
misc/checkers/check_load.py
View File

@ -1,5 +1,3 @@
from __future__ import print_function
""" """
this is a load-generating client program. It does all of its work through a this is a load-generating client program. It does all of its work through a
given tahoe node (specified by URL), and performs random reads and writes given tahoe node (specified by URL), and performs random reads and writes
@ -33,20 +31,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
""" """
from __future__ import annotations
import os, sys, httplib, binascii import os, sys, httplib, binascii
import urllib, json, random, time, urlparse import urllib, json, random, time, urlparse
try:
from typing import Dict
except ImportError:
pass
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401
if sys.argv[1] == "--stats": if sys.argv[1] == "--stats":
statsfiles = sys.argv[2:] statsfiles = sys.argv[2:]
# gather stats every 10 seconds, do a moving-window average of the last # gather stats every 10 seconds, do a moving-window average of the last
@ -54,9 +43,9 @@ if sys.argv[1] == "--stats":
DELAY = 10 DELAY = 10
MAXSAMPLES = 6 MAXSAMPLES = 6
totals = [] totals = []
last_stats = {} # type: Dict[str, float] last_stats : dict[str, float] = {}
while True: while True:
stats = {} # type: Dict[str, float] stats : dict[str, float] = {}
for sf in statsfiles: for sf in statsfiles:
for line in open(sf, "r").readlines(): for line in open(sf, "r").readlines():
name, str_value = line.split(":") name, str_value = line.split(":")

44
misc/coding_tools/find-trailing-spaces.py
View File

@ -1,44 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import os, sys
from twisted.python import usage
class Options(usage.Options):
optFlags = [
("recursive", "r", "Search for .py files recursively"),
]
def parseArgs(self, *starting_points):
self.starting_points = starting_points
found = [False]
def check(fn):
f = open(fn, "r")
for i,line in enumerate(f.readlines()):
if line == "\n":
continue
if line[-1] == "\n":
line = line[:-1]
if line.rstrip() != line:
# the %s:%d:%d: lets emacs' compile-mode jump to those locations
print("%s:%d:%d: trailing whitespace" % (fn, i+1, len(line)+1))
found[0] = True
f.close()
o = Options()
o.parseOptions()
if o['recursive']:
for starting_point in o.starting_points:
for root, dirs, files in os.walk(starting_point):
for fn in [f for f in files if f.endswith(".py")]:
fn = os.path.join(root, fn)
check(fn)
else:
for fn in o.starting_points:
check(fn)
if found[0]:
sys.exit(1)
sys.exit(0)

16
mypy.ini
View File

@ -7,4 +7,18 @@ show_error_codes = True
warn_unused_configs =True warn_unused_configs =True
no_implicit_optional = True no_implicit_optional = True
warn_redundant_casts = True warn_redundant_casts = True
strict_equality = True strict_equality = True
[mypy-allmydata.test.cli.wormholetesting,allmydata.test.test_connection_status]
disallow_any_generics = True
disallow_subclassing_any = True
disallow_untyped_calls = True
disallow_untyped_defs = True
disallow_incomplete_defs = True
check_untyped_defs = True
disallow_untyped_decorators = True
warn_unused_ignores = True
warn_return_any = True
no_implicit_reexport = True
strict_equality = True
strict_concatenate = True

0
newsfragments/3622.minor Normal file
View File

0
newsfragments/3880.minor Normal file
View File

0
newsfragments/3910.minor Normal file
View File

0
newsfragments/3935.minor Normal file
View File

0
newsfragments/3970.minor Normal file
View File

0
newsfragments/3978.minor Normal file
View File

0
newsfragments/3988.minor Normal file
View File

1
newsfragments/3989.installation Normal file
View File

@ -0,0 +1 @@
tenacity is no longer a dependency.

0
newsfragments/3991.minor Normal file
View File

0
newsfragments/3993.minor Normal file
View File

0
newsfragments/3994.minor Normal file
View File

0
newsfragments/3996.minor Normal file
View File

1
newsfragments/3997.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version.

0
newsfragments/3998.minor Normal file
View File

1
newsfragments/3999.bugfix Normal file
View File

@ -0,0 +1 @@
A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.

0
newsfragments/4000.minor Normal file
View File

0
newsfragments/4001.minor Normal file
View File

0
newsfragments/4002.minor Normal file
View File

0
newsfragments/4003.minor Normal file
View File

0
newsfragments/4005.minor Normal file
View File

0
newsfragments/4006.minor Normal file
View File

0
newsfragments/4009.minor Normal file
View File

0
newsfragments/4010.minor Normal file
View File

0
newsfragments/4012.minor Normal file
View File

0
newsfragments/4014.minor Normal file
View File

0
newsfragments/4015.minor Normal file
View File

0
newsfragments/4016.minor Normal file
View File

0
newsfragments/4018.minor Normal file
View File

0
newsfragments/4019.minor Normal file
View File

1
newsfragments/4020.minor Normal file
View File

@ -0,0 +1 @@

0
newsfragments/4022.minor Normal file
View File

0
newsfragments/4023.minor Normal file
View File

0
newsfragments/4024.minor Normal file
View File

0
newsfragments/4026.minor Normal file
View File

0
newsfragments/4027.minor Normal file
View File

0
newsfragments/4028.minor Normal file
View File

2
newsfragments/4029.bugfix Normal file
View File

@ -0,0 +1,2 @@
The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested.
Previously it would use normal TCP connections and not be anonymous.

0
newsfragments/4035.minor Normal file
View File

1
newsfragments/4036.feature Normal file
View File

@ -0,0 +1 @@
tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes"

12
nix/collections-extended.nix Normal file
View File

@ -0,0 +1,12 @@
# Package a version that's compatible with Python 3.11. This can go away once
# https://github.com/mlenzen/collections-extended/pull/199 is merged and
# included in a version of nixpkgs we depend on.
{ fetchFromGitHub, collections-extended }:
collections-extended.overrideAttrs (old: {
src = fetchFromGitHub {
owner = "mlenzen";
repo = "collections-extended";
rev = "8b93390636d58d28012b8e9d22334ee64ca37d73";
hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0=";
};
})

9
nix/klein.nix Normal file
View File

@ -0,0 +1,9 @@
{ klein, fetchPypi }:
klein.overrideAttrs (old: rec {
pname = "klein";
version = "23.5.0";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc=";
};
})

8
nix/pycddl.nix
View File

@ -27,7 +27,7 @@
# #
# 8. run `nix-build`. it should succeed. if it does not, seek assistance. # 8. run `nix-build`. it should succeed. if it does not, seek assistance.
# #
{ lib, fetchPypi, buildPythonPackage, rustPlatform }: { lib, fetchPypi, python, buildPythonPackage, rustPlatform }:
buildPythonPackage rec { buildPythonPackage rec {
pname = "pycddl"; pname = "pycddl";
version = "0.4.0"; version = "0.4.0";
@ -38,6 +38,12 @@ buildPythonPackage rec {
sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY="; sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
}; };
# Without this, when building for PyPy, `maturin build` seems to fail to
# find the interpreter at all and then fails early in the build process with
# an error saying "unsupported Python interpreter". We can easily point
# directly at the relevant interpreter, so do that.
maturinBuildFlags = [ "--interpreter" python.executable ];
nativeBuildInputs = with rustPlatform; [ nativeBuildInputs = with rustPlatform; [
maturinBuildHook maturinBuildHook
cargoSetupHook cargoSetupHook

10
nix/pyopenssl.nix Normal file
View File

@ -0,0 +1,10 @@
{ pyopenssl, fetchPypi, isPyPy }:
pyopenssl.overrideAttrs (old: rec {
pname = "pyOpenSSL";
version = "23.2.0";
name = "${pname}-${version}";
src = fetchPypi {
inherit pname version;
sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";
};
})

148
nix/python-overrides.nix Normal file
View File

@ -0,0 +1,148 @@
# Override various Python packages to create a package set that works for
# Tahoe-LAFS on CPython and PyPy.
self: super:
let
# Run a function on a derivation if and only if we're building for PyPy.
onPyPy = f: drv: if super.isPyPy then f drv else drv;
# Disable a Python package's test suite.
dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; });
# Disable building a Python package's documentation.
dontBuildDocs = alsoDisable: drv: (drv.override ({
sphinxHook = null;
} // alsoDisable)).overrideAttrs ({ outputs, ... }: {
outputs = builtins.filter (x: "doc" != x) outputs;
});
in {
# Some dependencies aren't packaged in nixpkgs so supply our own packages.
pycddl = self.callPackage ./pycddl.nix { };
txi2p = self.callPackage ./txi2p.nix { };
# Some packages are of somewhat too-old versions - update them.
klein = self.callPackage ./klein.nix {
# Avoid infinite recursion.
inherit (super) klein;
};
txtorcon = self.callPackage ./txtorcon.nix {
inherit (super) txtorcon;
};
# Update the version of pyopenssl.
pyopenssl = self.callPackage ./pyopenssl.nix {
pyopenssl =
# Building the docs requires sphinx which brings in a dependency on babel,
# the test suite of which fails.
onPyPy (dontBuildDocs { sphinx-rtd-theme = null; })
# Avoid infinite recursion.
super.pyopenssl;
};
# collections-extended is currently broken for Python 3.11 in nixpkgs but
# we know where a working version lives.
collections-extended = self.callPackage ./collections-extended.nix {
inherit (super) collections-extended;
};
# greenlet is incompatible with PyPy but PyPy has a builtin equivalent.
# Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67.
greenlet = onPyPy (drv: null) super.greenlet;
# tornado and tk pull in a huge dependency trees for functionality we don't
# care about, also tkinter doesn't work on PyPy.
matplotlib = super.matplotlib.override { tornado = null; enableTk = false; };
tqdm = super.tqdm.override {
# ibid.
tkinter = null;
# pandas is only required by the part of the test suite covering
# integration with pandas that we don't care about. pandas is a huge
# dependency.
pandas = null;
};
# The treq test suite depends on httpbin. httpbin pulls in babel (flask ->
# jinja2 -> babel) and arrow (brotlipy -> construct -> arrow). babel fails
# its test suite and arrow segfaults.
treq = onPyPy dontCheck super.treq;
# the six test suite fails on PyPy because it depends on dbm which the
# nixpkgs PyPy build appears to be missing. Maybe fixed in nixpkgs in
# a5f8184fb816a4fd5ae87136838c9981e0d22c67.
six = onPyPy dontCheck super.six;
# Likewise for beautifulsoup4.
beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
# The autobahn test suite pulls in a vast number of dependencies for
# functionality we don't care about. It might be nice to *selectively*
# disable just some of it but this is easier.
autobahn = onPyPy dontCheck super.autobahn;
# and python-dotenv tests pulls in a lot of dependencies, including jedi,
# which does not work on PyPy.
python-dotenv = onPyPy dontCheck super.python-dotenv;
# Upstream package unaccountably includes a sqlalchemy dependency ... but
# the project has no such dependency. Fixed in nixpkgs in
# da10e809fff70fbe1d86303b133b779f09f56503.
aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; };
# By default, the sphinx docs are built, which pulls in a lot of
# dependencies - including jedi, which does not work on PyPy.
hypothesis =
(let h = super.hypothesis;
in
if (h.override.__functionArgs.enableDocumentation or false)
then h.override { enableDocumentation = false; }
else h).overrideAttrs ({ nativeBuildInputs, ... }: {
# The nixpkgs expression is missing the tzdata check input.
nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ];
});
# flaky's test suite depends on nose and nose appears to have Python 3
# incompatibilities (it includes `print` statements, for example).
flaky = onPyPy dontCheck super.flaky;
# Replace the deprecated way of running the test suite with the modern way.
# This also drops a bunch of unnecessary build-time dependencies, some of
# which are broken on PyPy. Fixed in nixpkgs in
# 5feb5054bb08ba779bd2560a44cf7d18ddf37fea.
zfec = (super.zfec.override {
setuptoolsTrial = null;
}).overrideAttrs (old: {
checkPhase = "trial zfec";
});
# collections-extended is packaged with poetry-core. poetry-core test suite
# uses virtualenv and virtualenv test suite fails on PyPy.
poetry-core = onPyPy dontCheck super.poetry-core;
# The test suite fails with some rather irrelevant (to us) string comparison
# failure on PyPy. Probably a PyPy bug but doesn't seem like we should
# care.
rich = onPyPy dontCheck super.rich;
# The pyutil test suite fails in some ... test ... for some deprecation
# functionality we don't care about.
pyutil = onPyPy dontCheck super.pyutil;
# testCall1 fails fairly inscrutibly on PyPy. Perhaps someone can fix that,
# or we could at least just skip that one test. Probably better to fix it
# since we actually depend directly and significantly on Foolscap.
foolscap = onPyPy dontCheck super.foolscap;
# Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246
psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: {
# Upstream already disables some tests but there are even more that have
# build impurities that come from build system hardware configuration.
# Skip them too.
pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray;
disabledTests = disabledTests ++ [ "sensors_temperatures" ];
});
# CircleCI build systems don't have enough memory to run this test suite.
lz4 = dontCheck super.lz4;
}

12
nix/tahoe-lafs.nix
View File

@ -34,6 +34,7 @@ let
magic-wormhole magic-wormhole
netifaces netifaces
psutil psutil
pyyaml
pycddl pycddl
pyrsistent pyrsistent
pyutil pyutil
@ -48,20 +49,15 @@ let
zope_interface zope_interface
] ++ pickExtraDependencies pythonExtraDependencies extrasNames; ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
pythonCheckDependencies = with pythonPackages; [ unitTestDependencies = with pythonPackages; [
beautifulsoup4 beautifulsoup4
fixtures fixtures
hypothesis hypothesis
mock mock
paramiko
prometheus-client prometheus-client
pytest
pytest-timeout
pytest-twisted
tenacity
testtools testtools
towncrier
]; ];
in in
buildPythonPackage { buildPythonPackage {
inherit pname version; inherit pname version;
@ -69,7 +65,7 @@ buildPythonPackage {
propagatedBuildInputs = pythonPackageDependencies; propagatedBuildInputs = pythonPackageDependencies;
inherit doCheck; inherit doCheck;
checkInputs = pythonCheckDependencies; checkInputs = unitTestDependencies;
checkPhase = '' checkPhase = ''
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
python -m twisted.trial -j $NIX_BUILD_CORES allmydata python -m twisted.trial -j $NIX_BUILD_CORES allmydata

9
nix/txtorcon.nix Normal file
View File

@ -0,0 +1,9 @@
{ txtorcon, fetchPypi }:
txtorcon.overrideAttrs (old: rec {
pname = "txtorcon";
version = "23.5.0";
src = fetchPypi {
inherit pname version;
hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw=";
};
})

3
setup.cfg
View File

@ -6,6 +6,9 @@ develop = update_version develop
bdist_egg = update_version bdist_egg bdist_egg = update_version bdist_egg
bdist_wheel = update_version bdist_wheel bdist_wheel = update_version bdist_wheel
# This has been replaced by ruff (see .ruff.toml), which has same checks as
# flake8 plus many more, and is also faster. However, we're keeping this config
# in case people still use flake8 in IDEs, etc..
[flake8] [flake8]
# Enforce all pyflakes constraints, and also prohibit tabs for indentation. # Enforce all pyflakes constraints, and also prohibit tabs for indentation.
# Reference: # Reference:

46
setup.py
View File

@ -65,6 +65,9 @@ install_requires = [
# version of cryptography will *really* be installed. # version of cryptography will *really* be installed.
"cryptography >= 2.6", "cryptography >= 2.6",
# * Used for custom HTTPS validation
"pyOpenSSL >= 23.2.0",
# * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
# rekeying bug <https://twistedmatrix.com/trac/ticket/4395> # rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
# * The SFTP frontend and manhole depend on the conch extra. However, we # * The SFTP frontend and manhole depend on the conch extra. However, we
@ -136,7 +139,8 @@ install_requires = [
"collections-extended >= 2.0.2", "collections-extended >= 2.0.2",
# HTTP server and client # HTTP server and client
"klein", # Latest version is necessary to work with latest werkzeug:
"klein >= 23.5.0",
# 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
"werkzeug != 2.2.0", "werkzeug != 2.2.0",
"treq", "treq",
@ -159,10 +163,9 @@ setup_requires = [
] ]
tor_requires = [ tor_requires = [
# This is exactly what `foolscap[tor]` means but pip resolves the pair of # 23.5 added support for custom TLS contexts in web_agent(), which is
# dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose # needed for the HTTP storage client to run over Tor.
# this if we don't declare it ourselves! "txtorcon >= 23.5.0",
"txtorcon >= 0.17.0",
] ]
i2p_requires = [ i2p_requires = [
@ -394,16 +397,31 @@ setup(name="tahoe-lafs", # also set in __init__.py
"dulwich", "dulwich",
"gpg", "gpg",
], ],
# Here are the dependencies required to set up a reproducible test
# environment. This could be for CI or local development. These
# are *not* library dependencies of the test suite itself. They are
# the tools we use to run the test suite at all.
"testenv": [
# Pin all of these versions for the same reason you ever want to
# pin anything: to prevent new releases with regressions from
# introducing spurious failures into CI runs for whatever
# development work is happening at the time. The versions
# selected here are just the current versions at the time.
# Bumping them to keep up with future releases is fine as long
# as those releases are known to actually work.
"pip==22.0.3",
"wheel==0.37.1",
"setuptools==60.9.1",
"subunitreporter==22.2.0",
"python-subunit==1.4.2",
"junitxml==0.7",
"coverage==7.2.5",
],
# Here are the library dependencies of the test suite.
"test": [ "test": [
"flake8",
# Pin a specific pyflakes so we don't have different folks
# disagreeing on what is or is not a lint issue. We can bump
# this version from time to time, but we will do it
# intentionally.
"pyflakes == 2.2.0",
"coverage ~= 5.0",
"mock", "mock",
"tox ~= 3.0",
"pytest", "pytest",
"pytest-twisted", "pytest-twisted",
"hypothesis >= 3.6.1", "hypothesis >= 3.6.1",
@ -412,8 +430,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
"fixtures", "fixtures",
"beautifulsoup4", "beautifulsoup4",
"html5lib", "html5lib",
"junitxml",
"tenacity",
# Pin old version until # Pin old version until
# https://github.com/paramiko/paramiko/issues/1961 is fixed. # https://github.com/paramiko/paramiko/issues/1961 is fixed.
"paramiko < 2.9", "paramiko < 2.9",

17
src/allmydata/client.py
View File

@ -7,10 +7,9 @@ import os
import stat import stat
import time import time
import weakref import weakref
from typing import Optional from typing import Optional, Iterable
from base64 import urlsafe_b64encode from base64 import urlsafe_b64encode
from functools import partial from functools import partial
# On Python 2 this will be the backported package:
from configparser import NoSectionError from configparser import NoSectionError
from foolscap.furl import ( from foolscap.furl import (
@ -47,7 +46,7 @@ from allmydata.util.encodingutil import get_filesystem_encoding
from allmydata.util.abbreviate import parse_abbreviated_size from allmydata.util.abbreviate import parse_abbreviated_size
from allmydata.util.time_format import parse_duration, parse_date from allmydata.util.time_format import parse_duration, parse_date
from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.i2p_provider import create as create_i2p_provider
from allmydata.util.tor_provider import create as create_tor_provider from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider
from allmydata.stats import StatsProvider from allmydata.stats import StatsProvider
from allmydata.history import History from allmydata.history import History
from allmydata.interfaces import ( from allmydata.interfaces import (
@ -175,8 +174,6 @@ class KeyGenerator(object):
"""I return a Deferred that fires with a (verifyingkey, signingkey) """I return a Deferred that fires with a (verifyingkey, signingkey)
pair. The returned key will be 2048 bit""" pair. The returned key will be 2048 bit"""
keysize = 2048 keysize = 2048
# RSA key generation for a 2048 bit key takes between 0.8 and 3.2
# secs
signer, verifier = rsa.create_signing_keypair(keysize) signer, verifier = rsa.create_signing_keypair(keysize)
return defer.succeed( (verifier, signer) ) return defer.succeed( (verifier, signer) )
@ -191,7 +188,7 @@ class Terminator(service.Service):
return service.Service.stopService(self) return service.Service.stopService(self)
def read_config(basedir, portnumfile, generated_files=[]): def read_config(basedir, portnumfile, generated_files: Iterable=()):
""" """
Read and validate configuration for a client-style Node. See Read and validate configuration for a client-style Node. See
:method:`allmydata.node.read_config` for parameter meanings (the :method:`allmydata.node.read_config` for parameter meanings (the
@ -270,7 +267,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory=
introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory)
storage_broker = create_storage_farm_broker( storage_broker = create_storage_farm_broker(
config, default_connection_handlers, foolscap_connection_handlers, config, default_connection_handlers, foolscap_connection_handlers,
tub_options, introducer_clients tub_options, introducer_clients, tor_provider
) )
client = _client_factory( client = _client_factory(
@ -466,7 +463,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
return introducer_clients return introducer_clients
def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients): def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients, tor_provider: Optional[TorProvider]):
""" """
Create a StorageFarmBroker object, for use by Uploader/Downloader Create a StorageFarmBroker object, for use by Uploader/Downloader
(and everybody else who wants to use storage servers) (and everybody else who wants to use storage servers)
@ -502,6 +499,8 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo
tub_maker=tub_creator, tub_maker=tub_creator,
node_config=config, node_config=config,
storage_client_config=storage_client_config, storage_client_config=storage_client_config,
default_connection_handlers=default_connection_handlers,
tor_provider=tor_provider,
) )
for ic in introducer_clients: for ic in introducer_clients:
sb.use_introducer(ic) sb.use_introducer(ic)
@ -1105,7 +1104,7 @@ class _Client(node.Node, pollmixin.PollMixin):
# may get an opaque node if there were any problems. # may get an opaque node if there were any problems.
return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
def create_dirnode(self, initial_children={}, version=None): def create_dirnode(self, initial_children=None, version=None):
d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
return d return d

4
src/allmydata/dirnode.py
View File

@ -678,8 +678,10 @@ class DirectoryNode(object):
return d return d
# XXX: Too many arguments? Worthwhile to break into mutable/immutable? # XXX: Too many arguments? Worthwhile to break into mutable/immutable?
def create_subdirectory(self, namex, initial_children={}, overwrite=True, def create_subdirectory(self, namex, initial_children=None, overwrite=True,
mutable=True, mutable_version=None, metadata=None): mutable=True, mutable_version=None, metadata=None):
if initial_children is None:
initial_children = {}
name = normalize(namex) name = normalize(namex)
if self.is_readonly(): if self.is_readonly():
return defer.fail(NotWriteableError()) return defer.fail(NotWriteableError())

17
src/allmydata/frontends/sftpd.py
View File

@ -1925,7 +1925,11 @@ class FakeTransport(object):
def loseConnection(self): def loseConnection(self):
logmsg("FakeTransport.loseConnection()", level=NOISY) logmsg("FakeTransport.loseConnection()", level=NOISY)
# getPeer and getHost can just raise errors, since we don't know what to return def getHost(self):
raise NotImplementedError()
def getPeer(self):
raise NotImplementedError()
@implementer(ISession) @implementer(ISession)
@ -1990,15 +1994,18 @@ class Dispatcher(object):
def __init__(self, client): def __init__(self, client):
self._client = client self._client = client
def requestAvatar(self, avatarID, mind, interface): def requestAvatar(self, avatarId, mind, *interfaces):
[interface] = interfaces
_assert(interface == IConchUser, interface=interface) _assert(interface == IConchUser, interface=interface)
rootnode = self._client.create_node_from_uri(avatarID.rootcap) rootnode = self._client.create_node_from_uri(avatarId.rootcap)
handler = SFTPUserHandler(self._client, rootnode, avatarID.username) handler = SFTPUserHandler(self._client, rootnode, avatarId.username)
return (interface, handler, handler.logout) return (interface, handler, handler.logout)
class SFTPServer(service.MultiService): class SFTPServer(service.MultiService):
name = "frontend:sftp" # The type in Twisted for services is wrong in 22.10...
# https://github.com/twisted/twisted/issues/10135
name = "frontend:sftp" # type: ignore[assignment]
def __init__(self, client, accountfile, def __init__(self, client, accountfile,
sftp_portstr, pubkey_file, privkey_file): sftp_portstr, pubkey_file, privkey_file):

7
src/allmydata/hashtree.py
View File

@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
name += " (leaf [%d] of %d)" % (leafnum, numleaves) name += " (leaf [%d] of %d)" % (leafnum, numleaves)
return name return name
def set_hashes(self, hashes={}, leaves={}): def set_hashes(self, hashes=None, leaves=None):
"""Add a bunch of hashes to the tree. """Add a bunch of hashes to the tree.
I will validate these to the best of my ability. If I already have a I will validate these to the best of my ability. If I already have a
@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
corrupted or one of the received hashes was corrupted. If it raises corrupted or one of the received hashes was corrupted. If it raises
NotEnoughHashesError, then the otherhashes dictionary was incomplete. NotEnoughHashesError, then the otherhashes dictionary was incomplete.
""" """
if hashes is None:
hashes = {}
if leaves is None:
leaves = {}
assert isinstance(hashes, dict) assert isinstance(hashes, dict)
for h in hashes.values(): for h in hashes.values():
assert isinstance(h, bytes) assert isinstance(h, bytes)

28
src/allmydata/immutable/upload.py
View File

@ -2,22 +2,12 @@
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import absolute_import from __future__ import annotations
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, native_str from future.utils import native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long, unicode from past.builtins import long, unicode
from six import ensure_str from six import ensure_str
try:
from typing import List
except ImportError:
pass
import os, time, weakref, itertools import os, time, weakref, itertools
import attr import attr
@ -915,12 +905,12 @@ class _Accum(object):
:ivar remaining: The number of bytes still expected. :ivar remaining: The number of bytes still expected.
:ivar ciphertext: The bytes accumulated so far. :ivar ciphertext: The bytes accumulated so far.
""" """
remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int remaining : int = attr.ib(validator=attr.validators.instance_of(int))
ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes] ciphertext : list[bytes] = attr.ib(default=attr.Factory(list))
def extend(self, def extend(self,
size, # type: int size, # type: int
ciphertext, # type: List[bytes] ciphertext, # type: list[bytes]
): ):
""" """
Accumulate some more ciphertext. Accumulate some more ciphertext.
@ -1401,7 +1391,9 @@ class CHKUploader(object):
def get_upload_status(self): def get_upload_status(self):
return self._upload_status return self._upload_status
def read_this_many_bytes(uploadable, size, prepend_data=[]): def read_this_many_bytes(uploadable, size, prepend_data=None):
if prepend_data is None:
prepend_data = []
if size == 0: if size == 0:
return defer.succeed([]) return defer.succeed([])
d = uploadable.read(size) d = uploadable.read(size)
@ -1851,7 +1843,9 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
"""I am a service that allows file uploading. I am a service-child of the """I am a service that allows file uploading. I am a service-child of the
Client. Client.
""" """
name = "uploader" # The type in Twisted for services is wrong in 22.10...
# https://github.com/twisted/twisted/issues/10135
name = "uploader" # type: ignore[assignment]
URI_LIT_SIZE_THRESHOLD = 55 URI_LIT_SIZE_THRESHOLD = 55
def __init__(self, helper_furl=None, stats_provider=None, history=None): def __init__(self, helper_furl=None, stats_provider=None, history=None):

64
src/allmydata/interfaces.py
View File

@ -17,11 +17,13 @@ if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401 from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min # noqa: F401
from past.builtins import long from past.builtins import long
from typing import Dict
from zope.interface import Interface, Attribute from zope.interface import Interface, Attribute
from twisted.plugin import ( from twisted.plugin import (
IPlugin, IPlugin,
) )
from twisted.internet.defer import Deferred
from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \ from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable
@ -307,12 +309,15 @@ class RIStorageServer(RemoteInterface):
store that on disk. store that on disk.
""" """
# The result of IStorageServer.get_version():
VersionMessage = Dict[bytes, object]
class IStorageServer(Interface): class IStorageServer(Interface):
""" """
An object capable of storing shares for a storage client. An object capable of storing shares for a storage client.
""" """
def get_version(): def get_version() -> Deferred[VersionMessage]:
""" """
:see: ``RIStorageServer.get_version`` :see: ``RIStorageServer.get_version``
""" """
@ -493,47 +498,6 @@ class IStorageBroker(Interface):
@return: unicode nickname, or None @return: unicode nickname, or None
""" """
# methods moved from IntroducerClient, need review
def get_all_connections():
"""Return a frozenset of (nodeid, service_name, rref) tuples, one for
each active connection we've established to a remote service. This is
mostly useful for unit tests that need to wait until a certain number
of connections have been made."""
def get_all_connectors():
"""Return a dict that maps from (nodeid, service_name) to a
RemoteServiceConnector instance for all services that we are actively
trying to connect to. Each RemoteServiceConnector has the following
public attributes::
service_name: the type of service provided, like 'storage'
last_connect_time: when we last established a connection
last_loss_time: when we last lost a connection
version: the peer's version, from the most recent connection
oldest_supported: the peer's oldest supported version, same
rref: the RemoteReference, if connected, otherwise None
This method is intended for monitoring interfaces, such as a web page
that describes connecting and connected peers.
"""
def get_all_peerids():
"""Return a frozenset of all peerids to whom we have a connection (to
one or more services) established. Mostly useful for unit tests."""
def get_all_connections_for(service_name):
"""Return a frozenset of (nodeid, service_name, rref) tuples, one
for each active connection that provides the given SERVICE_NAME."""
def get_permuted_peers(service_name, key):
"""Returns an ordered list of (peerid, rref) tuples, selecting from
the connections that provide SERVICE_NAME, using a hash-based
permutation keyed by KEY. This randomizes the service list in a
repeatable way, to distribute load over many peers.
"""
class IDisplayableServer(Interface): class IDisplayableServer(Interface):
def get_nickname(): def get_nickname():
@ -551,16 +515,6 @@ class IServer(IDisplayableServer):
def start_connecting(trigger_cb): def start_connecting(trigger_cb):
pass pass
def get_rref():
"""Obsolete. Use ``get_storage_server`` instead.
Once a server is connected, I return a RemoteReference.
Before a server is connected for the first time, I return None.
Note that the rref I return will start producing DeadReferenceErrors
once the connection is lost.
"""
def upload_permitted(): def upload_permitted():
""" """
:return: True if we should use this server for uploads, False :return: True if we should use this server for uploads, False
@ -1447,7 +1401,7 @@ class IDirectoryNode(IFilesystemNode):
is a file, or if must_be_file is True and the child is a directory, is a file, or if must_be_file is True and the child is a directory,
I raise ChildOfWrongTypeError.""" I raise ChildOfWrongTypeError."""
def create_subdirectory(name, initial_children={}, overwrite=True, def create_subdirectory(name, initial_children=None, overwrite=True,
mutable=True, mutable_version=None, metadata=None): mutable=True, mutable_version=None, metadata=None):
"""I create and attach a directory at the given name. The new """I create and attach a directory at the given name. The new
directory can be empty, or it can be populated with children directory can be empty, or it can be populated with children
@ -2586,7 +2540,7 @@ class IClient(Interface):
@return: a Deferred that fires with an IMutableFileNode instance. @return: a Deferred that fires with an IMutableFileNode instance.
""" """
def create_dirnode(initial_children={}): def create_dirnode(initial_children=None):
"""Create a new unattached dirnode, possibly with initial children. """Create a new unattached dirnode, possibly with initial children.
@param initial_children: dict with keys that are unicode child names, @param initial_children: dict with keys that are unicode child names,
@ -2641,7 +2595,7 @@ class INodeMaker(Interface):
for use by unit tests, to create mutable files that are smaller than for use by unit tests, to create mutable files that are smaller than
usual.""" usual."""
def create_new_mutable_directory(initial_children={}): def create_new_mutable_directory(initial_children=None):
"""I create a new mutable directory, and return a Deferred that will """I create a new mutable directory, and return a Deferred that will
fire with the IDirectoryNode instance when it is ready. If fire with the IDirectoryNode instance when it is ready. If
initial_children= is provided (a dict mapping unicode child name to initial_children= is provided (a dict mapping unicode child name to

2
src/allmydata/introducer/client.py
View File

@ -35,7 +35,7 @@ class InvalidCacheError(Exception):
V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2" V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2"
@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient) @implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient) # type: ignore[misc]
class IntroducerClient(service.Service, Referenceable): class IntroducerClient(service.Service, Referenceable):
def __init__(self, tub, introducer_furl, def __init__(self, tub, introducer_furl,

32
src/allmydata/introducer/server.py
View File

@ -2,24 +2,13 @@
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import absolute_import from __future__ import annotations
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long from past.builtins import long
from six import ensure_text from six import ensure_text
import time, os.path, textwrap import time, os.path, textwrap
from typing import Any, Union
try:
from typing import Any, Dict, Union
except ImportError:
pass
from zope.interface import implementer from zope.interface import implementer
from twisted.application import service from twisted.application import service
@ -79,10 +68,6 @@ def create_introducer(basedir=u"."):
default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider) default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider)
tub_options = create_tub_options(config) tub_options = create_tub_options(config)
# we don't remember these because the Introducer doesn't make
# outbound connections.
i2p_provider = None
tor_provider = None
main_tub = create_main_tub( main_tub = create_main_tub(
config, tub_options, default_connection_handlers, config, tub_options, default_connection_handlers,
foolscap_connection_handlers, i2p_provider, tor_provider, foolscap_connection_handlers, i2p_provider, tor_provider,
@ -94,6 +79,8 @@ def create_introducer(basedir=u"."):
i2p_provider, i2p_provider,
tor_provider, tor_provider,
) )
i2p_provider.setServiceParent(node)
tor_provider.setServiceParent(node)
return defer.succeed(node) return defer.succeed(node)
except Exception: except Exception:
return Failure() return Failure()
@ -155,17 +142,20 @@ def stringify_remote_address(rref):
return str(remote) return str(remote)
# MyPy doesn't work well with remote interfaces...
@implementer(RIIntroducerPublisherAndSubscriberService_v2) @implementer(RIIntroducerPublisherAndSubscriberService_v2)
class IntroducerService(service.MultiService, Referenceable): class IntroducerService(service.MultiService, Referenceable): # type: ignore[misc]
name = "introducer" # The type in Twisted for services is wrong in 22.10...
# https://github.com/twisted/twisted/issues/10135
name = "introducer" # type: ignore[assignment]
# v1 is the original protocol, added in 1.0 (but only advertised starting # v1 is the original protocol, added in 1.0 (but only advertised starting
# in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
# TODO: reconcile bytes/str for keys # TODO: reconcile bytes/str for keys
VERSION = { VERSION : dict[Union[bytes, str], Any]= {
#"http://allmydata.org/tahoe/protocols/introducer/v1": { }, #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
b"application-version": allmydata.__full_version__.encode("utf-8"), b"application-version": allmydata.__full_version__.encode("utf-8"),
} # type: Dict[Union[bytes, str], Any] }
def __init__(self): def __init__(self):
service.MultiService.__init__(self) service.MultiService.__init__(self)

27
src/allmydata/node.py
View File

@ -4,14 +4,8 @@ a node for Tahoe-LAFS.
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import absolute_import from __future__ import annotations
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_str, ensure_text from six import ensure_str, ensure_text
import json import json
@ -23,11 +17,7 @@ import errno
from base64 import b32decode, b32encode from base64 import b32decode, b32encode
from errno import ENOENT, EPERM from errno import ENOENT, EPERM
from warnings import warn from warnings import warn
from typing import Union, Iterable
try:
from typing import Union
except ImportError:
pass
import attr import attr
@ -182,7 +172,7 @@ def create_node_dir(basedir, readme_text):
f.write(readme_text) f.write(readme_text)
def read_config(basedir, portnumfile, generated_files=[], _valid_config=None): def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None):
""" """
Read and validate configuration. Read and validate configuration.
@ -281,8 +271,7 @@ def _error_about_old_config_files(basedir, generated_files):
raise e raise e
def ensure_text_and_abspath_expanduser_unicode(basedir): def ensure_text_and_abspath_expanduser_unicode(basedir: Union[bytes, str]) -> str:
# type: (Union[bytes, str]) -> str
return abspath_expanduser_unicode(ensure_text(basedir)) return abspath_expanduser_unicode(ensure_text(basedir))
@ -752,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider):
def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers, def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
handler_overrides={}, force_foolscap=False, **kwargs): handler_overrides=None, force_foolscap=False, **kwargs):
""" """
Create a Tub with the right options and handlers. It will be Create a Tub with the right options and handlers. It will be
ephemeral unless the caller provides certFile= in kwargs ephemeral unless the caller provides certFile= in kwargs
@ -766,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
:param bool force_foolscap: If True, only allow Foolscap, not just HTTPS :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS
storage protocol. storage protocol.
""" """
if handler_overrides is None:
handler_overrides = {}
# We listen simultaneously for both Foolscap and HTTPS on the same port, # We listen simultaneously for both Foolscap and HTTPS on the same port,
# so we have to create a special Foolscap Tub for that to work: # so we have to create a special Foolscap Tub for that to work:
if force_foolscap: if force_foolscap:
@ -933,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location):
def create_main_tub(config, tub_options, def create_main_tub(config, tub_options,
default_connection_handlers, foolscap_connection_handlers, default_connection_handlers, foolscap_connection_handlers,
i2p_provider, tor_provider, i2p_provider, tor_provider,
handler_overrides={}, cert_filename="node.pem"): handler_overrides=None, cert_filename="node.pem"):
""" """
Creates a 'main' Foolscap Tub, typically for use as the top-level Creates a 'main' Foolscap Tub, typically for use as the top-level
access point for a running Node. access point for a running Node.
@ -954,6 +945,8 @@ def create_main_tub(config, tub_options,
:param tor_provider: None, or a _Provider instance if txtorcon + :param tor_provider: None, or a _Provider instance if txtorcon +
Tor are installed. Tor are installed.
""" """
if handler_overrides is None:
handler_overrides = {}
portlocation = _tub_portlocation( portlocation = _tub_portlocation(
config, config,
iputil.get_local_addresses_sync, iputil.get_local_addresses_sync,

5
src/allmydata/nodemaker.py
View File

@ -135,8 +135,9 @@ class NodeMaker(object):
d.addCallback(lambda res: n) d.addCallback(lambda res: n)
return d return d
def create_new_mutable_directory(self, initial_children={}, version=None): def create_new_mutable_directory(self, initial_children=None, version=None):
# initial_children must have metadata (i.e. {} instead of None) if initial_children is None:
initial_children = {}
for (name, (node, metadata)) in initial_children.items(): for (name, (node, metadata)) in initial_children.items():
precondition(isinstance(metadata, dict), precondition(isinstance(metadata, dict),
"create_new_mutable_directory requires metadata to be a dict, not None", metadata) "create_new_mutable_directory requires metadata to be a dict, not None", metadata)

19
src/allmydata/protocol_switch.py
View File

@ -16,9 +16,10 @@ later in the configuration process.
from __future__ import annotations from __future__ import annotations
from itertools import chain from itertools import chain
from typing import cast
from twisted.internet.protocol import Protocol from twisted.internet.protocol import Protocol
from twisted.internet.interfaces import IDelayedCall from twisted.internet.interfaces import IDelayedCall, IReactorFromThreads
from twisted.internet.ssl import CertificateOptions from twisted.internet.ssl import CertificateOptions
from twisted.web.server import Site from twisted.web.server import Site
from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.protocols.tls import TLSMemoryBIOFactory
@ -89,7 +90,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
certificate=cls.tub.myCertificate.original, certificate=cls.tub.myCertificate.original,
) )
http_storage_server = HTTPServer(reactor, storage_server, swissnum) http_storage_server = HTTPServer(cast(IReactorFromThreads, reactor), storage_server, swissnum)
cls.https_factory = TLSMemoryBIOFactory( cls.https_factory = TLSMemoryBIOFactory(
certificate_options, certificate_options,
False, False,
@ -102,8 +103,15 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
for location_hint in chain.from_iterable( for location_hint in chain.from_iterable(
hints.split(",") for hints in cls.tub.locationHints hints.split(",") for hints in cls.tub.locationHints
): ):
if location_hint.startswith("tcp:"): if location_hint.startswith("tcp:") or location_hint.startswith("tor:"):
_, hostname, port = location_hint.split(":") scheme, hostname, port = location_hint.split(":")
if scheme == "tcp":
subscheme = None
else:
subscheme = "tor"
# If we're listening on Tor, the hostname needs to have an
# .onion TLD.
assert hostname.endswith(".onion")
port = int(port) port = int(port)
storage_nurls.add( storage_nurls.add(
build_nurl( build_nurl(
@ -111,9 +119,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
port, port,
str(swissnum, "ascii"), str(swissnum, "ascii"),
cls.tub.myCertificate.original.to_cryptography(), cls.tub.myCertificate.original.to_cryptography(),
subscheme
) )
) )
# TODO this is probably where we'll have to support Tor and I2P? # TODO this is where we'll have to support Tor and I2P as well.
# See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9 # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9
# for discussion (there will be separate tickets added for those at # for discussion (there will be separate tickets added for those at
# some point.) # some point.)

11
src/allmydata/scripts/admin.py
View File

@ -112,6 +112,9 @@ class AddGridManagerCertOptions(BaseOptions):
return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]" return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]"
def postOptions(self) -> None: def postOptions(self) -> None:
assert self.parent is not None
assert self.parent.parent is not None
if self['name'] is None: if self['name'] is None:
raise usage.UsageError( raise usage.UsageError(
"Must provide --name option" "Must provide --name option"
@ -123,8 +126,8 @@ class AddGridManagerCertOptions(BaseOptions):
data: str data: str
if self['filename'] == '-': if self['filename'] == '-':
print("reading certificate from stdin", file=self.parent.parent.stderr) print("reading certificate from stdin", file=self.parent.parent.stderr) # type: ignore[attr-defined]
data = self.parent.parent.stdin.read() data = self.parent.parent.stdin.read() # type: ignore[attr-defined]
if len(data) == 0: if len(data) == 0:
raise usage.UsageError( raise usage.UsageError(
"Reading certificate from stdin failed" "Reading certificate from stdin failed"
@ -255,9 +258,9 @@ def do_admin(options):
return f(so) return f(so)
subCommands = [ subCommands : SubCommands = [
("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"),
] # type: SubCommands ]
dispatch = { dispatch = {
"admin": do_admin, "admin": do_admin,

22
src/allmydata/scripts/cli.py
View File

@ -1,22 +1,10 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os.path, re, fnmatch import os.path, re, fnmatch
try: from allmydata.scripts.types_ import SubCommands, Parameters
from allmydata.scripts.types_ import SubCommands, Parameters
except ImportError:
pass
from twisted.python import usage from twisted.python import usage
from allmydata.scripts.common import get_aliases, get_default_nodedir, \ from allmydata.scripts.common import get_aliases, get_default_nodedir, \
@ -29,14 +17,14 @@ NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
_default_nodedir = get_default_nodedir() _default_nodedir = get_default_nodedir()
class FileStoreOptions(BaseOptions): class FileStoreOptions(BaseOptions):
optParameters = [ optParameters : Parameters = [
["node-url", "u", None, ["node-url", "u", None,
"Specify the URL of the Tahoe gateway node, such as " "Specify the URL of the Tahoe gateway node, such as "
"'http://127.0.0.1:3456'. " "'http://127.0.0.1:3456'. "
"This overrides the URL found in the --node-directory ."], "This overrides the URL found in the --node-directory ."],
["dir-cap", None, None, ["dir-cap", None, None,
"Specify which dirnode URI should be used as the 'tahoe' alias."] "Specify which dirnode URI should be used as the 'tahoe' alias."]
] # type: Parameters ]
def postOptions(self): def postOptions(self):
self["quiet"] = self.parent["quiet"] self["quiet"] = self.parent["quiet"]
@ -484,7 +472,7 @@ class DeepCheckOptions(FileStoreOptions):
(which must be a directory), like 'tahoe check' but for multiple files. (which must be a directory), like 'tahoe check' but for multiple files.
Optionally repair any problems found.""" Optionally repair any problems found."""
subCommands = [ subCommands : SubCommands = [
("mkdir", None, MakeDirectoryOptions, "Create a new directory."), ("mkdir", None, MakeDirectoryOptions, "Create a new directory."),
("add-alias", None, AddAliasOptions, "Add a new alias cap."), ("add-alias", None, AddAliasOptions, "Add a new alias cap."),
("create-alias", None, CreateAliasOptions, "Create a new alias cap."), ("create-alias", None, CreateAliasOptions, "Create a new alias cap."),
@ -503,7 +491,7 @@ subCommands = [
("check", None, CheckOptions, "Check a single file or directory."), ("check", None, CheckOptions, "Check a single file or directory."),
("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."),
("status", None, TahoeStatusCommand, "Various status information."), ("status", None, TahoeStatusCommand, "Various status information."),
] # type: SubCommands ]
def mkdir(options): def mkdir(options):
from allmydata.scripts import tahoe_mkdir from allmydata.scripts import tahoe_mkdir

36
src/allmydata/scripts/common.py
View File

@ -4,29 +4,13 @@
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import unicode_literals from typing import Union, Optional
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
else:
from typing import Union
import os, sys, textwrap import os, sys, textwrap
import codecs import codecs
from os.path import join from os.path import join
import urllib.parse import urllib.parse
try:
from typing import Optional
from .types_ import Parameters
except ImportError:
pass
from yaml import ( from yaml import (
safe_dump, safe_dump,
) )
@ -37,6 +21,8 @@ from allmydata.util.assertutil import precondition
from allmydata.util.encodingutil import quote_output, \ from allmydata.util.encodingutil import quote_output, \
quote_local_unicode_path, argv_to_abspath quote_local_unicode_path, argv_to_abspath
from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.scripts.default_nodedir import _default_nodedir
from .types_ import Parameters
def get_default_nodedir(): def get_default_nodedir():
return _default_nodedir return _default_nodedir
@ -59,7 +45,7 @@ class BaseOptions(usage.Options):
def opt_version(self): def opt_version(self):
raise usage.UsageError("--version not allowed on subcommands") raise usage.UsageError("--version not allowed on subcommands")
description = None # type: Optional[str] description : Optional[str] = None
description_unwrapped = None # type: Optional[str] description_unwrapped = None # type: Optional[str]
def __str__(self): def __str__(self):
@ -80,10 +66,10 @@ class BaseOptions(usage.Options):
class BasedirOptions(BaseOptions): class BasedirOptions(BaseOptions):
default_nodedir = _default_nodedir default_nodedir = _default_nodedir
optParameters = [ optParameters : Parameters = [
["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
% quote_local_unicode_path(_default_nodedir)], % quote_local_unicode_path(_default_nodedir)],
] # type: Parameters ]
def parseArgs(self, basedir=None): def parseArgs(self, basedir=None):
# This finds the node-directory option correctly even if we are in a subcommand. # This finds the node-directory option correctly even if we are in a subcommand.
@ -283,9 +269,8 @@ def get_alias(aliases, path_unicode, default):
quote_output(alias)) quote_output(alias))
return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
def escape_path(path): def escape_path(path: Union[str, bytes]) -> str:
# type: (Union[str,bytes]) -> str """
u"""
Return path quoted to US-ASCII, valid URL characters. Return path quoted to US-ASCII, valid URL characters.
>>> path = u'/føö/bar/☃' >>> path = u'/føö/bar/☃'
@ -302,9 +287,4 @@ def escape_path(path):
]), ]),
"ascii" "ascii"
) )
# Eventually (i.e. as part of Python 3 port) we want this to always return
# Unicode strings. However, to reduce diff sizes in the short term it'll
# return native string (i.e. bytes) on Python 2.
if PY2:
result = result.encode("ascii").__native__()
return result return result

37
src/allmydata/scripts/common_http.py
View File

@ -1,19 +1,11 @@
""" """
Ported to Python 3. Blocking HTTP client APIs.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os import os
from io import BytesIO from io import BytesIO
from six.moves import urllib, http_client from http import client as http_client
import six import urllib
import allmydata # for __full_version__ import allmydata # for __full_version__
from allmydata.util.encodingutil import quote_output from allmydata.util.encodingutil import quote_output
@ -51,7 +43,7 @@ class BadResponse(object):
def do_http(method, url, body=b""): def do_http(method, url, body=b""):
if isinstance(body, bytes): if isinstance(body, bytes):
body = BytesIO(body) body = BytesIO(body)
elif isinstance(body, six.text_type): elif isinstance(body, str):
raise TypeError("do_http body must be a bytestring, not unicode") raise TypeError("do_http body must be a bytestring, not unicode")
else: else:
# We must give a Content-Length header to twisted.web, otherwise it # We must give a Content-Length header to twisted.web, otherwise it
@ -61,10 +53,17 @@ def do_http(method, url, body=b""):
assert body.seek assert body.seek
assert body.read assert body.read
scheme, host, port, path = parse_url(url) scheme, host, port, path = parse_url(url)
# For testing purposes, allow setting a timeout on HTTP requests. If this
# ever become a user-facing feature, this should probably be a CLI option?
timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None)
if timeout is not None:
timeout = float(timeout)
if scheme == "http": if scheme == "http":
c = http_client.HTTPConnection(host, port) c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536)
elif scheme == "https": elif scheme == "https":
c = http_client.HTTPSConnection(host, port) c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536)
else: else:
raise ValueError("unknown scheme '%s', need http or https" % scheme) raise ValueError("unknown scheme '%s', need http or https" % scheme)
c.putrequest(method, path) c.putrequest(method, path)
@ -85,7 +84,7 @@ def do_http(method, url, body=b""):
return BadResponse(url, err) return BadResponse(url, err)
while True: while True:
data = body.read(8192) data = body.read(65536)
if not data: if not data:
break break
c.send(data) c.send(data)
@ -94,16 +93,14 @@ def do_http(method, url, body=b""):
def format_http_success(resp): def format_http_success(resp):
# ensure_text() shouldn't be necessary when Python 2 is dropped.
return quote_output( return quote_output(
"%s %s" % (resp.status, six.ensure_text(resp.reason)), "%s %s" % (resp.status, resp.reason),
quotemarks=False) quotemarks=False)
def format_http_error(msg, resp): def format_http_error(msg, resp):
# ensure_text() shouldn't be necessary when Python 2 is dropped.
return quote_output( return quote_output(
"%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason), "%s: %s %s\n%r" % (msg, resp.status, resp.reason,
six.ensure_text(resp.read())), resp.read()),
quotemarks=False) quotemarks=False)
def check_http_error(resp, stderr): def check_http_error(resp, stderr):

48
src/allmydata/scripts/create_node.py
View File

@ -1,25 +1,11 @@
# Ported to Python 3
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import io import io
import os import os
try: from allmydata.scripts.types_ import (
from allmydata.scripts.types_ import ( SubCommands,
SubCommands, Parameters,
Parameters, Flags,
Flags, )
)
except ImportError:
pass
from twisted.internet import reactor, defer from twisted.internet import reactor, defer
from twisted.python.usage import UsageError from twisted.python.usage import UsageError
@ -48,7 +34,7 @@ def write_tac(basedir, nodetype):
fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac) fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac)
WHERE_OPTS = [ WHERE_OPTS : Parameters = [
("location", None, None, ("location", None, None,
"Server location to advertise (e.g. tcp:example.org:12345)"), "Server location to advertise (e.g. tcp:example.org:12345)"),
("port", None, None, ("port", None, None,
@ -57,29 +43,29 @@ WHERE_OPTS = [
"Hostname to automatically set --location/--port when --listen=tcp"), "Hostname to automatically set --location/--port when --listen=tcp"),
("listen", None, "tcp", ("listen", None, "tcp",
"Comma-separated list of listener types (tcp,tor,i2p,none)."), "Comma-separated list of listener types (tcp,tor,i2p,none)."),
] # type: Parameters ]
TOR_OPTS = [ TOR_OPTS : Parameters = [
("tor-control-port", None, None, ("tor-control-port", None, None,
"Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"), "Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"),
("tor-executable", None, None, ("tor-executable", None, None,
"The 'tor' executable to run (default is to search $PATH)."), "The 'tor' executable to run (default is to search $PATH)."),
] # type: Parameters ]
TOR_FLAGS = [ TOR_FLAGS : Flags = [
("tor-launch", None, "Launch a tor instead of connecting to a tor control port."), ("tor-launch", None, "Launch a tor instead of connecting to a tor control port."),
] # type: Flags ]
I2P_OPTS = [ I2P_OPTS : Parameters = [
("i2p-sam-port", None, None, ("i2p-sam-port", None, None,
"I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"), "I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"),
("i2p-executable", None, None, ("i2p-executable", None, None,
"(future) The 'i2prouter' executable to run (default is to search $PATH)."), "(future) The 'i2prouter' executable to run (default is to search $PATH)."),
] # type: Parameters ]
I2P_FLAGS = [ I2P_FLAGS : Flags = [
("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."), ("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."),
] # type: Flags ]
def validate_where_options(o): def validate_where_options(o):
if o['listen'] == "none": if o['listen'] == "none":
@ -508,11 +494,11 @@ def create_introducer(config):
defer.returnValue(0) defer.returnValue(0)
subCommands = [ subCommands : SubCommands = [
("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."),
("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."),
("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."),
] # type: SubCommands ]
dispatch = { dispatch = {
"create-node": create_node, "create-node": create_node,

18
src/allmydata/scripts/debug.py
View File

@ -1,19 +1,8 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2, bchr from future.utils import bchr
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
import struct, time, os, sys import struct, time, os, sys
@ -31,6 +20,7 @@ from allmydata.mutable.common import NeedMoreDataError
from allmydata.immutable.layout import ReadBucketProxy from allmydata.immutable.layout import ReadBucketProxy
from allmydata.util import base32 from allmydata.util import base32
from allmydata.util.encodingutil import quote_output from allmydata.util.encodingutil import quote_output
from allmydata.scripts.types_ import SubCommands
class DumpOptions(BaseOptions): class DumpOptions(BaseOptions):
def getSynopsis(self): def getSynopsis(self):
@ -1076,9 +1066,9 @@ def do_debug(options):
return f(so) return f(so)
subCommands = [ subCommands : SubCommands = [
("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
] # type: SubCommands ]
dispatch = { dispatch = {
"debug": do_debug, "debug": do_debug,

21
src/allmydata/scripts/runner.py
View File

@ -1,28 +1,15 @@
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os, sys import os, sys
from six.moves import StringIO from io import StringIO
from past.builtins import unicode from past.builtins import unicode
import six import six
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
from twisted.python import usage from twisted.python import usage
from twisted.internet import defer, task, threads from twisted.internet import defer, task, threads
from allmydata.scripts.common import get_default_nodedir from allmydata.scripts.common import get_default_nodedir
from allmydata.scripts import debug, create_node, cli, \ from allmydata.scripts import debug, create_node, cli, \
admin, tahoe_run, tahoe_invite admin, tahoe_run, tahoe_invite
from allmydata.scripts.types_ import SubCommands
from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode
from allmydata.util.eliotutil import ( from allmydata.util.eliotutil import (
opt_eliot_destination, opt_eliot_destination,
@ -47,9 +34,9 @@ if _default_nodedir:
NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
process_control_commands = [ process_control_commands : SubCommands = [
("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"),
] # type: SubCommands ]
class Options(usage.Options): class Options(usage.Options):

18
src/allmydata/scripts/tahoe_invite.py
View File

@ -1,19 +1,6 @@
""" """
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
from twisted.python import usage from twisted.python import usage
from twisted.internet import defer, reactor from twisted.internet import defer, reactor
@ -21,6 +8,7 @@ from twisted.internet import defer, reactor
from allmydata.util.encodingutil import argv_to_abspath from allmydata.util.encodingutil import argv_to_abspath
from allmydata.util import jsonbytes as json from allmydata.util import jsonbytes as json
from allmydata.scripts.common import get_default_nodedir, get_introducer_furl from allmydata.scripts.common import get_default_nodedir, get_introducer_furl
from allmydata.scripts.types_ import SubCommands
from allmydata.client import read_config from allmydata.client import read_config
@ -112,10 +100,10 @@ def invite(options):
print("Completed successfully", file=out) print("Completed successfully", file=out)
subCommands = [ subCommands : SubCommands = [
("invite", None, InviteOptions, ("invite", None, InviteOptions,
"Invite a new node to this grid"), "Invite a new node to this grid"),
] # type: SubCommands ]
dispatch = { dispatch = {
"invite": invite, "invite": invite,

20
src/allmydata/scripts/tahoe_run.py
View File

@ -104,6 +104,11 @@ class RunOptions(BasedirOptions):
" [default: %s]" % quote_local_unicode_path(_default_nodedir)), " [default: %s]" % quote_local_unicode_path(_default_nodedir)),
] ]
optFlags = [
("allow-stdin-close", None,
'Do not exit when stdin closes ("tahoe run" otherwise will exit).'),
]
def parseArgs(self, basedir=None, *twistd_args): def parseArgs(self, basedir=None, *twistd_args):
# This can't handle e.g. 'tahoe run --reactor=foo', since # This can't handle e.g. 'tahoe run --reactor=foo', since
# '--reactor=foo' looks like an option to the tahoe subcommand, not to # '--reactor=foo' looks like an option to the tahoe subcommand, not to
@ -156,6 +161,7 @@ class DaemonizeTheRealService(Service, HookMixin):
"running": None, "running": None,
} }
self.stderr = options.parent.stderr self.stderr = options.parent.stderr
self._close_on_stdin_close = False if options["allow-stdin-close"] else True
def startService(self): def startService(self):
@ -199,10 +205,12 @@ class DaemonizeTheRealService(Service, HookMixin):
d = service_factory() d = service_factory()
def created(srv): def created(srv):
srv.setServiceParent(self.parent) if self.parent is not None:
srv.setServiceParent(self.parent)
# exiting on stdin-closed facilitates cleanup when run # exiting on stdin-closed facilitates cleanup when run
# as a subprocess # as a subprocess
on_stdin_close(reactor, reactor.stop) if self._close_on_stdin_close:
on_stdin_close(reactor, reactor.stop)
d.addCallback(created) d.addCallback(created)
d.addErrback(handle_config_error) d.addErrback(handle_config_error)
d.addBoth(self._call_hook, 'running') d.addBoth(self._call_hook, 'running')
@ -213,11 +221,13 @@ class DaemonizeTheRealService(Service, HookMixin):
class DaemonizeTahoeNodePlugin(object): class DaemonizeTahoeNodePlugin(object):
tapname = "tahoenode" tapname = "tahoenode"
def __init__(self, nodetype, basedir): def __init__(self, nodetype, basedir, allow_stdin_close):
self.nodetype = nodetype self.nodetype = nodetype
self.basedir = basedir self.basedir = basedir
self.allow_stdin_close = allow_stdin_close
def makeService(self, so): def makeService(self, so):
so["allow-stdin-close"] = self.allow_stdin_close
return DaemonizeTheRealService(self.nodetype, self.basedir, so) return DaemonizeTheRealService(self.nodetype, self.basedir, so)
@ -304,7 +314,9 @@ def run(reactor, config, runApp=twistd.runApp):
print(config, file=err) print(config, file=err)
print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err) print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err)
return 1 return 1
twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} twistd_config.loadedPlugins = {
"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir, config["allow-stdin-close"])
}
# our own pid-style file contains PID and process creation time # our own pid-style file contains PID and process creation time
pidfile = FilePath(get_pidfile(config['basedir'])) pidfile = FilePath(get_pidfile(config['basedir']))

4
src/allmydata/storage/common.py
View File

@ -39,6 +39,10 @@ def si_b2a(storageindex):
def si_a2b(ascii_storageindex): def si_a2b(ascii_storageindex):
return base32.a2b(ascii_storageindex) return base32.a2b(ascii_storageindex)
def si_to_human_readable(storageindex: bytes) -> str:
"""Create human-readable string of storage index."""
return str(base32.b2a(storageindex), "ascii")
def storage_index_to_dir(storageindex): def storage_index_to_dir(storageindex):
"""Convert storage index to directory path. """Convert storage index to directory path.

667
src/allmydata/storage/http_client.py
View File

@ -4,12 +4,27 @@ HTTP client that talks to the HTTP storage server.
from __future__ import annotations from __future__ import annotations
from typing import Union, Optional, Sequence, Mapping, BinaryIO
from typing import (
Union,
Optional,
Sequence,
Mapping,
BinaryIO,
cast,
TypedDict,
Set,
Dict,
Callable,
ClassVar,
)
from base64 import b64encode from base64 import b64encode
from io import BytesIO from io import BytesIO
from os import SEEK_END from os import SEEK_END
from attrs import define, asdict, frozen, field from attrs import define, asdict, frozen, field
from eliot import start_action, register_exception_extractor
from eliot.twisted import DeferredContext
# TODO Make sure to import Python version? # TODO Make sure to import Python version?
from cbor2 import loads, dumps from cbor2 import loads, dumps
@ -18,8 +33,8 @@ from collections_extended import RangeMap
from werkzeug.datastructures import Range, ContentRange from werkzeug.datastructures import Range, ContentRange
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web import http from twisted.web import http
from twisted.web.iweb import IPolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS, IResponse, IAgent
from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed from twisted.internet.defer import Deferred, succeed
from twisted.internet.interfaces import ( from twisted.internet.interfaces import (
IOpenSSLClientConnectionCreator, IOpenSSLClientConnectionCreator,
IReactorTime, IReactorTime,
@ -33,7 +48,6 @@ import treq
from treq.client import HTTPClient from treq.client import HTTPClient
from treq.testing import StubTreq from treq.testing import StubTreq
from OpenSSL import SSL from OpenSSL import SSL
from cryptography.hazmat.bindings.openssl.binding import Binding
from werkzeug.http import parse_content_range_header from werkzeug.http import parse_content_range_header
from .http_common import ( from .http_common import (
@ -42,12 +56,20 @@ from .http_common import (
get_content_type, get_content_type,
CBOR_MIME_TYPE, CBOR_MIME_TYPE,
get_spki_hash, get_spki_hash,
response_is_not_html,
) )
from .common import si_b2a from ..interfaces import VersionMessage
from .common import si_b2a, si_to_human_readable
from ..util.hashutil import timing_safe_compare from ..util.hashutil import timing_safe_compare
from ..util.deferredutil import async_to_deferred from ..util.deferredutil import async_to_deferred
from ..util.tor_provider import _Provider as TorProvider
_OPENSSL = Binding().lib try:
from txtorcon import Tor # type: ignore
except ImportError:
class Tor: # type: ignore[no-redef]
pass
def _encode_si(si): # type: (bytes) -> str def _encode_si(si): # type: (bytes) -> str
@ -63,6 +85,9 @@ class ClientException(Exception):
self.code = code self.code = code
register_exception_extractor(ClientException, lambda e: {"response_code": e.code})
# Schemas for server responses. # Schemas for server responses.
# #
# Tags are of the form #6.nnn, where the number is documented at # Tags are of the form #6.nnn, where the number is documented at
@ -70,15 +95,14 @@ class ClientException(Exception):
# indicates a set. # indicates a set.
_SCHEMAS = { _SCHEMAS = {
"get_version": Schema( "get_version": Schema(
# Note that the single-quoted (`'`) string keys in this schema
# represent *byte* strings - per the CDDL specification. Text strings
# are represented using strings with *double* quotes (`"`).
""" """
response = {'http://allmydata.org/tahoe/protocols/storage/v1' => { response = {'http://allmydata.org/tahoe/protocols/storage/v1' => {
'maximum-immutable-share-size' => uint 'maximum-immutable-share-size' => uint
'maximum-mutable-share-size' => uint 'maximum-mutable-share-size' => uint
'available-space' => uint 'available-space' => uint
'tolerates-immutable-read-overrun' => bool
'delete-mutable-shares-with-zero-length-writev' => bool
'fills-holes-with-zero-bytes' => bool
'prevents-read-past-end-of-share-data' => bool
} }
'application-version' => bstr 'application-version' => bstr
} }
@ -156,15 +180,24 @@ def limited_content(
This will time out if no data is received for 60 seconds; so long as a This will time out if no data is received for 60 seconds; so long as a
trickle of data continues to arrive, it will continue to run. trickle of data continues to arrive, it will continue to run.
""" """
d = succeed(None) result_deferred = succeed(None)
timeout = clock.callLater(60, d.cancel)
# Sadly, addTimeout() won't work because we need access to the IDelayedCall
# in order to reset it on each data chunk received.
timeout = clock.callLater(60, result_deferred.cancel)
collector = _LengthLimitedCollector(max_length, timeout) collector = _LengthLimitedCollector(max_length, timeout)
with start_action(
action_type="allmydata:storage:http-client:limited-content",
max_length=max_length,
).context():
d = DeferredContext(result_deferred)
# Make really sure everything gets called in Deferred context, treq might # Make really sure everything gets called in Deferred context, treq might
# call collector directly... # call collector directly...
d.addCallback(lambda _: treq.collect(response, collector)) d.addCallback(lambda _: treq.collect(response, collector))
def done(_): def done(_: object) -> BytesIO:
timeout.cancel() timeout.cancel()
collector.f.seek(0) collector.f.seek(0)
return collector.f return collector.f
@ -174,7 +207,8 @@ def limited_content(
timeout.cancel() timeout.cancel()
return f return f
return d.addCallbacks(done, failed) result = d.addCallbacks(done, failed)
return result.addActionFinish()
@define @define
@ -231,11 +265,11 @@ class _TLSContextFactory(CertificateOptions):
# not the usual TLS concerns about invalid CAs or revoked # not the usual TLS concerns about invalid CAs or revoked
# certificates. # certificates.
things_are_ok = ( things_are_ok = (
_OPENSSL.X509_V_OK, SSL.X509VerificationCodes.OK,
_OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID, SSL.X509VerificationCodes.ERR_CERT_NOT_YET_VALID,
_OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED, SSL.X509VerificationCodes.ERR_CERT_HAS_EXPIRED,
_OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT, SSL.X509VerificationCodes.ERR_DEPTH_ZERO_SELF_SIGNED_CERT,
_OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN, SSL.X509VerificationCodes.ERR_SELF_SIGNED_CERT_IN_CHAIN,
) )
# TODO can we do this once instead of multiple times? # TODO can we do this once instead of multiple times?
if errno in things_are_ok and timing_safe_compare( if errno in things_are_ok and timing_safe_compare(
@ -276,18 +310,30 @@ class _StorageClientHTTPSPolicy:
) )
@define(hash=True) @define
class StorageClient(object): class StorageClientFactory:
""" """
Low-level HTTP client that talks to the HTTP storage server. Create ``StorageClient`` instances, using appropriate
``twisted.web.iweb.IAgent`` for different connection methods: normal TCP,
Tor, and eventually I2P.
There is some caching involved since there might be shared setup work, e.g.
connecting to the local Tor service only needs to happen once.
""" """
# If set, we're doing unit testing and we should call this with _default_connection_handlers: dict[str, str]
# HTTPConnectionPool we create. _tor_provider: Optional[TorProvider]
TEST_MODE_REGISTER_HTTP_POOL = None # Cache the Tor instance created by the provider, if relevant.
_tor_instance: Optional[Tor] = None
# If set, we're doing unit testing and we should call this with any
# HTTPConnectionPool that gets passed/created to ``create_agent()``.
TEST_MODE_REGISTER_HTTP_POOL: ClassVar[
Optional[Callable[[HTTPConnectionPool], None]]
] = None
@classmethod @classmethod
def start_test_mode(cls, callback): def start_test_mode(cls, callback: Callable[[HTTPConnectionPool], None]) -> None:
"""Switch to testing mode. """Switch to testing mode.
In testing mode we register the pool with test system using the given In testing mode we register the pool with test system using the given
@ -302,44 +348,90 @@ class StorageClient(object):
"""Stop testing mode.""" """Stop testing mode."""
cls.TEST_MODE_REGISTER_HTTP_POOL = None cls.TEST_MODE_REGISTER_HTTP_POOL = None
# The URL is a HTTPS URL ("https://..."). To construct from a NURL, use async def _create_agent(
# ``StorageClient.from_nurl()``. self,
nurl: DecodedURL,
reactor: object,
tls_context_factory: IPolicyForHTTPS,
pool: HTTPConnectionPool,
) -> IAgent:
"""Create a new ``IAgent``, possibly using Tor."""
if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
self.TEST_MODE_REGISTER_HTTP_POOL(pool)
# TODO default_connection_handlers should really be an object, not a
# dict, so we can ask "is this using Tor" without poking at a
# dictionary with arbitrary strings... See
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032
handler = self._default_connection_handlers["tcp"]
if handler == "tcp":
return Agent(reactor, tls_context_factory, pool=pool)
if handler == "tor" or nurl.scheme == "pb+tor":
assert self._tor_provider is not None
if self._tor_instance is None:
self._tor_instance = await self._tor_provider.get_tor_instance(reactor)
return self._tor_instance.web_agent(
pool=pool, tls_context_factory=tls_context_factory
)
else:
raise RuntimeError(f"Unsupported tcp connection handler: {handler}")
async def create_storage_client(
self,
nurl: DecodedURL,
reactor: IReactorTime,
pool: Optional[HTTPConnectionPool] = None,
) -> StorageClient:
"""Create a new ``StorageClient`` for the given NURL."""
assert nurl.fragment == "v=1"
assert nurl.scheme in ("pb", "pb+tor")
if pool is None:
pool = HTTPConnectionPool(reactor)
pool.maxPersistentPerHost = 10
certificate_hash = nurl.user.encode("ascii")
agent = await self._create_agent(
nurl,
reactor,
_StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
pool,
)
treq_client = HTTPClient(agent)
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
swissnum = nurl.path[0].encode("ascii")
response_check = lambda _: None
if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
response_check = response_is_not_html
return StorageClient(
https_url,
swissnum,
treq_client,
pool,
reactor,
response_check,
)
@define(hash=True)
class StorageClient(object):
"""
Low-level HTTP client that talks to the HTTP storage server.
Create using a ``StorageClientFactory`` instance.
"""
# The URL should be a HTTPS URL ("https://...")
_base_url: DecodedURL _base_url: DecodedURL
_swissnum: bytes _swissnum: bytes
_treq: Union[treq, StubTreq, HTTPClient] _treq: Union[treq, StubTreq, HTTPClient]
_pool: HTTPConnectionPool
_clock: IReactorTime _clock: IReactorTime
# Are we running unit tests?
_analyze_response: Callable[[IResponse], None] = lambda _: None
@classmethod def relative_url(self, path: str) -> DecodedURL:
def from_nurl(
cls,
nurl: DecodedURL,
reactor,
) -> StorageClient:
"""
Create a ``StorageClient`` for the given NURL.
"""
assert nurl.fragment == "v=1"
assert nurl.scheme == "pb"
swissnum = nurl.path[0].encode("ascii")
certificate_hash = nurl.user.encode("ascii")
pool = HTTPConnectionPool(reactor)
pool.maxPersistentPerHost = 20
if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
treq_client = HTTPClient(
Agent(
reactor,
_StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
pool=pool,
)
)
https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
return cls(https_url, swissnum, treq_client, reactor)
def relative_url(self, path):
"""Get a URL relative to the base URL.""" """Get a URL relative to the base URL."""
return self._base_url.click(path) return self._base_url.click(path)
@ -353,19 +445,20 @@ class StorageClient(object):
) )
return headers return headers
def request( @async_to_deferred
async def request(
self, self,
method, method: str,
url, url: DecodedURL,
lease_renew_secret=None, lease_renew_secret: Optional[bytes] = None,
lease_cancel_secret=None, lease_cancel_secret: Optional[bytes] = None,
upload_secret=None, upload_secret: Optional[bytes] = None,
write_enabler_secret=None, write_enabler_secret: Optional[bytes] = None,
headers=None, headers: Optional[Headers] = None,
message_to_serialize=None, message_to_serialize: object = None,
timeout: float = 60, timeout: float = 60,
**kwargs, **kwargs,
): ) -> IResponse:
""" """
Like ``treq.request()``, but with optional secrets that get translated Like ``treq.request()``, but with optional secrets that get translated
into corresponding HTTP headers. into corresponding HTTP headers.
@ -375,6 +468,41 @@ class StorageClient(object):
Default timeout is 60 seconds. Default timeout is 60 seconds.
""" """
with start_action(
action_type="allmydata:storage:http-client:request",
method=method,
url=url.to_text(),
timeout=timeout,
) as ctx:
response = await self._request(
method,
url,
lease_renew_secret,
lease_cancel_secret,
upload_secret,
write_enabler_secret,
headers,
message_to_serialize,
timeout,
**kwargs,
)
ctx.add_success_fields(response_code=response.code)
return response
async def _request(
self,
method: str,
url: DecodedURL,
lease_renew_secret: Optional[bytes] = None,
lease_cancel_secret: Optional[bytes] = None,
upload_secret: Optional[bytes] = None,
write_enabler_secret: Optional[bytes] = None,
headers: Optional[Headers] = None,
message_to_serialize: object = None,
timeout: float = 60,
**kwargs,
) -> IResponse:
"""The implementation of request()."""
headers = self._get_headers(headers) headers = self._get_headers(headers)
# Add secrets: # Add secrets:
@ -405,28 +533,39 @@ class StorageClient(object):
kwargs["data"] = dumps(message_to_serialize) kwargs["data"] = dumps(message_to_serialize)
headers.addRawHeader("Content-Type", CBOR_MIME_TYPE) headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
return self._treq.request( response = await self._treq.request(
method, url, headers=headers, timeout=timeout, **kwargs method, url, headers=headers, timeout=timeout, **kwargs
) )
self._analyze_response(response)
def decode_cbor(self, response, schema: Schema): return response
async def decode_cbor(self, response: IResponse, schema: Schema) -> object:
"""Given HTTP response, return decoded CBOR body.""" """Given HTTP response, return decoded CBOR body."""
with start_action(action_type="allmydata:storage:http-client:decode-cbor"):
def got_content(f: BinaryIO): if response.code > 199 and response.code < 300:
data = f.read() content_type = get_content_type(response.headers)
schema.validate_cbor(data) if content_type == CBOR_MIME_TYPE:
return loads(data) f = await limited_content(response, self._clock)
data = f.read()
if response.code > 199 and response.code < 300: schema.validate_cbor(data)
content_type = get_content_type(response.headers) return loads(data)
if content_type == CBOR_MIME_TYPE: else:
return limited_content(response, self._clock).addCallback(got_content) raise ClientException(
-1,
"Server didn't send CBOR, content type is {}".format(
content_type
),
)
else: else:
raise ClientException(-1, "Server didn't send CBOR") data = (
else: await limited_content(response, self._clock, max_length=10_000)
return treq.content(response).addCallback( ).read()
lambda data: fail(ClientException(response.code, response.phrase, data)) raise ClientException(response.code, response.phrase, data)
)
def shutdown(self) -> Deferred:
"""Shutdown any connections."""
return self._pool.closeCachedConnections()
@define(hash=True) @define(hash=True)
@ -437,32 +576,65 @@ class StorageClientGeneral(object):
_client: StorageClient _client: StorageClient
@inlineCallbacks @async_to_deferred
def get_version(self): async def get_version(self) -> VersionMessage:
""" """
Return the version metadata for the server. Return the version metadata for the server.
""" """
url = self._client.relative_url("/storage/v1/version") with start_action(
response = yield self._client.request("GET", url) action_type="allmydata:storage:http-client:get-version",
decoded_response = yield self._client.decode_cbor( ):
response, _SCHEMAS["get_version"] return await self._get_version()
)
returnValue(decoded_response)
@inlineCallbacks async def _get_version(self) -> VersionMessage:
def add_or_renew_lease( """Implementation of get_version()."""
url = self._client.relative_url("/storage/v1/version")
response = await self._client.request("GET", url)
decoded_response = cast(
Dict[bytes, object],
await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
)
# Add some features we know are true because the HTTP API
# specification requires them and because other parts of the storage
# client implementation assumes they will be present.
cast(
Dict[bytes, object],
decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
).update(
{
b"tolerates-immutable-read-overrun": True,
b"delete-mutable-shares-with-zero-length-writev": True,
b"fills-holes-with-zero-bytes": True,
b"prevents-read-past-end-of-share-data": True,
}
)
return decoded_response
@async_to_deferred
async def add_or_renew_lease(
self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes
) -> Deferred[None]: ) -> None:
""" """
Add or renew a lease. Add or renew a lease.
If the renewal secret matches an existing lease, it is renewed. If the renewal secret matches an existing lease, it is renewed.
Otherwise a new lease is added. Otherwise a new lease is added.
""" """
with start_action(
action_type="allmydata:storage:http-client:add-or-renew-lease",
storage_index=si_to_human_readable(storage_index),
):
return await self._add_or_renew_lease(
storage_index, renew_secret, cancel_secret
)
async def _add_or_renew_lease(
self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes
) -> None:
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/lease/{}".format(_encode_si(storage_index)) "/storage/v1/lease/{}".format(_encode_si(storage_index))
) )
response = yield self._client.request( response = await self._client.request(
"PUT", "PUT",
url, url,
lease_renew_secret=renew_secret, lease_renew_secret=renew_secret,
@ -487,15 +659,15 @@ class UploadProgress(object):
required: RangeMap required: RangeMap
@inlineCallbacks @async_to_deferred
def read_share_chunk( async def read_share_chunk(
client: StorageClient, client: StorageClient,
share_type: str, share_type: str,
storage_index: bytes, storage_index: bytes,
share_number: int, share_number: int,
offset: int, offset: int,
length: int, length: int,
) -> Deferred[bytes]: ) -> bytes:
""" """
Download a chunk of data from a share. Download a chunk of data from a share.
@ -516,7 +688,7 @@ def read_share_chunk(
# The default 60 second timeout is for getting the response, so it doesn't # The default 60 second timeout is for getting the response, so it doesn't
# include the time it takes to download the body... so we will will deal # include the time it takes to download the body... so we will will deal
# with that later, via limited_content(). # with that later, via limited_content().
response = yield client.request( response = await client.request(
"GET", "GET",
url, url,
headers=Headers( headers=Headers(
@ -530,6 +702,12 @@ def read_share_chunk(
if response.code == http.NO_CONTENT: if response.code == http.NO_CONTENT:
return b"" return b""
content_type = get_content_type(response.headers)
if content_type != "application/octet-stream":
raise ValueError(
f"Content-type was wrong: {content_type}, should be application/octet-stream"
)
if response.code == http.PARTIAL_CONTENT: if response.code == http.PARTIAL_CONTENT:
content_range = parse_content_range_header( content_range = parse_content_range_header(
response.headers.getRawHeaders("content-range")[0] or "" response.headers.getRawHeaders("content-range")[0] or ""
@ -547,7 +725,7 @@ def read_share_chunk(
raise ValueError("Server sent more than we asked for?!") raise ValueError("Server sent more than we asked for?!")
# It might also send less than we asked for. That's (probably) OK, e.g. # It might also send less than we asked for. That's (probably) OK, e.g.
# if we went past the end of the file. # if we went past the end of the file.
body = yield limited_content(response, client._clock, supposed_length) body = await limited_content(response, client._clock, supposed_length)
body.seek(0, SEEK_END) body.seek(0, SEEK_END)
actual_length = body.tell() actual_length = body.tell()
if actual_length != supposed_length: if actual_length != supposed_length:
@ -573,7 +751,7 @@ async def advise_corrupt_share(
storage_index: bytes, storage_index: bytes,
share_number: int, share_number: int,
reason: str, reason: str,
): ) -> None:
assert isinstance(reason, str) assert isinstance(reason, str)
url = client.relative_url( url = client.relative_url(
"/storage/v1/{}/{}/{}/corrupt".format( "/storage/v1/{}/{}/{}/corrupt".format(
@ -598,16 +776,16 @@ class StorageClientImmutables(object):
_client: StorageClient _client: StorageClient
@inlineCallbacks @async_to_deferred
def create( async def create(
self, self,
storage_index, storage_index: bytes,
share_numbers, share_numbers: set[int],
allocated_size, allocated_size: int,
upload_secret, upload_secret: bytes,
lease_renew_secret, lease_renew_secret: bytes,
lease_cancel_secret, lease_cancel_secret: bytes,
): # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult] ) -> ImmutableCreateResult:
""" """
Create a new storage index for an immutable. Create a new storage index for an immutable.
@ -621,12 +799,41 @@ class StorageClientImmutables(object):
Result fires when creating the storage index succeeded, if creating the Result fires when creating the storage index succeeded, if creating the
storage index failed the result will fire with an exception. storage index failed the result will fire with an exception.
""" """
with start_action(
action_type="allmydata:storage:http-client:immutable:create",
storage_index=si_to_human_readable(storage_index),
share_numbers=share_numbers,
allocated_size=allocated_size,
) as ctx:
result = await self._create(
storage_index,
share_numbers,
allocated_size,
upload_secret,
lease_renew_secret,
lease_cancel_secret,
)
ctx.add_success_fields(
already_have=result.already_have, allocated=result.allocated
)
return result
async def _create(
self,
storage_index: bytes,
share_numbers: set[int],
allocated_size: int,
upload_secret: bytes,
lease_renew_secret: bytes,
lease_cancel_secret: bytes,
) -> ImmutableCreateResult:
"""Implementation of create()."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/immutable/" + _encode_si(storage_index) "/storage/v1/immutable/" + _encode_si(storage_index)
) )
message = {"share-numbers": share_numbers, "allocated-size": allocated_size} message = {"share-numbers": share_numbers, "allocated-size": allocated_size}
response = yield self._client.request( response = await self._client.request(
"POST", "POST",
url, url,
lease_renew_secret=lease_renew_secret, lease_renew_secret=lease_renew_secret,
@ -634,27 +841,37 @@ class StorageClientImmutables(object):
upload_secret=upload_secret, upload_secret=upload_secret,
message_to_serialize=message, message_to_serialize=message,
) )
decoded_response = yield self._client.decode_cbor( decoded_response = cast(
response, _SCHEMAS["allocate_buckets"] Mapping[str, Set[int]],
await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]),
) )
returnValue( return ImmutableCreateResult(
ImmutableCreateResult( already_have=decoded_response["already-have"],
already_have=decoded_response["already-have"], allocated=decoded_response["allocated"],
allocated=decoded_response["allocated"],
)
) )
@inlineCallbacks @async_to_deferred
def abort_upload( async def abort_upload(
self, storage_index: bytes, share_number: int, upload_secret: bytes self, storage_index: bytes, share_number: int, upload_secret: bytes
) -> Deferred[None]: ) -> None:
"""Abort the upload.""" """Abort the upload."""
with start_action(
action_type="allmydata:storage:http-client:immutable:abort-upload",
storage_index=si_to_human_readable(storage_index),
share_number=share_number,
):
return await self._abort_upload(storage_index, share_number, upload_secret)
async def _abort_upload(
self, storage_index: bytes, share_number: int, upload_secret: bytes
) -> None:
"""Implementation of ``abort_upload()``."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/immutable/{}/{}/abort".format( "/storage/v1/immutable/{}/{}/abort".format(
_encode_si(storage_index), share_number _encode_si(storage_index), share_number
) )
) )
response = yield self._client.request( response = await self._client.request(
"PUT", "PUT",
url, url,
upload_secret=upload_secret, upload_secret=upload_secret,
@ -667,10 +884,15 @@ class StorageClientImmutables(object):
response.code, response.code,
) )
@inlineCallbacks @async_to_deferred
def write_share_chunk( async def write_share_chunk(
self, storage_index, share_number, upload_secret, offset, data self,
): # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress] storage_index: bytes,
share_number: int,
upload_secret: bytes,
offset: int,
data: bytes,
) -> UploadProgress:
""" """
Upload a chunk of data for a specific share. Upload a chunk of data for a specific share.
@ -683,12 +905,34 @@ class StorageClientImmutables(object):
whether the _complete_ share (i.e. all chunks, not just this one) has whether the _complete_ share (i.e. all chunks, not just this one) has
been uploaded. been uploaded.
""" """
with start_action(
action_type="allmydata:storage:http-client:immutable:write-share-chunk",
storage_index=si_to_human_readable(storage_index),
share_number=share_number,
offset=offset,
data_len=len(data),
) as ctx:
result = await self._write_share_chunk(
storage_index, share_number, upload_secret, offset, data
)
ctx.add_success_fields(finished=result.finished)
return result
async def _write_share_chunk(
self,
storage_index: bytes,
share_number: int,
upload_secret: bytes,
offset: int,
data: bytes,
) -> UploadProgress:
"""Implementation of ``write_share_chunk()``."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/immutable/{}/{}".format( "/storage/v1/immutable/{}/{}".format(
_encode_si(storage_index), share_number _encode_si(storage_index), share_number
) )
) )
response = yield self._client.request( response = await self._client.request(
"PATCH", "PATCH",
url, url,
upload_secret=upload_secret, upload_secret=upload_secret,
@ -712,52 +956,84 @@ class StorageClientImmutables(object):
raise ClientException( raise ClientException(
response.code, response.code,
) )
body = yield self._client.decode_cbor( body = cast(
response, _SCHEMAS["immutable_write_share_chunk"] Mapping[str, Sequence[Mapping[str, int]]],
await self._client.decode_cbor(
response, _SCHEMAS["immutable_write_share_chunk"]
),
) )
remaining = RangeMap() remaining = RangeMap()
for chunk in body["required"]: for chunk in body["required"]:
remaining.set(True, chunk["begin"], chunk["end"]) remaining.set(True, chunk["begin"], chunk["end"])
returnValue(UploadProgress(finished=finished, required=remaining)) return UploadProgress(finished=finished, required=remaining)
def read_share_chunk( @async_to_deferred
self, storage_index, share_number, offset, length async def read_share_chunk(
): # type: (bytes, int, int, int) -> Deferred[bytes] self, storage_index: bytes, share_number: int, offset: int, length: int
) -> bytes:
""" """
Download a chunk of data from a share. Download a chunk of data from a share.
""" """
return read_share_chunk( with start_action(
self._client, "immutable", storage_index, share_number, offset, length action_type="allmydata:storage:http-client:immutable:read-share-chunk",
) storage_index=si_to_human_readable(storage_index),
share_number=share_number,
offset=offset,
length=length,
) as ctx:
result = await read_share_chunk(
self._client, "immutable", storage_index, share_number, offset, length
)
ctx.add_success_fields(data_len=len(result))
return result
@inlineCallbacks @async_to_deferred
def list_shares(self, storage_index: bytes) -> Deferred[set[int]]: async def list_shares(self, storage_index: bytes) -> Set[int]:
""" """
Return the set of shares for a given storage index. Return the set of shares for a given storage index.
""" """
with start_action(
action_type="allmydata:storage:http-client:immutable:list-shares",
storage_index=si_to_human_readable(storage_index),
) as ctx:
result = await self._list_shares(storage_index)
ctx.add_success_fields(shares=result)
return result
async def _list_shares(self, storage_index: bytes) -> Set[int]:
"""Implementation of ``list_shares()``."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/immutable/{}/shares".format(_encode_si(storage_index)) "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
) )
response = yield self._client.request( response = await self._client.request(
"GET", "GET",
url, url,
) )
if response.code == http.OK: if response.code == http.OK:
body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"]) return cast(
returnValue(set(body)) Set[int],
await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
)
else: else:
raise ClientException(response.code) raise ClientException(response.code)
def advise_corrupt_share( @async_to_deferred
async def advise_corrupt_share(
self, self,
storage_index: bytes, storage_index: bytes,
share_number: int, share_number: int,
reason: str, reason: str,
): ) -> None:
"""Indicate a share has been corrupted, with a human-readable message.""" """Indicate a share has been corrupted, with a human-readable message."""
return advise_corrupt_share( with start_action(
self._client, "immutable", storage_index, share_number, reason action_type="allmydata:storage:http-client:immutable:advise-corrupt-share",
) storage_index=si_to_human_readable(storage_index),
share_number=share_number,
reason=reason,
):
await advise_corrupt_share(
self._client, "immutable", storage_index, share_number, reason
)
@frozen @frozen
@ -814,6 +1090,13 @@ class ReadTestWriteResult:
reads: Mapping[int, Sequence[bytes]] reads: Mapping[int, Sequence[bytes]]
# Result type for mutable read/test/write HTTP response. Can't just use
# dict[int,list[bytes]] because on Python 3.8 that will error out.
MUTABLE_RTW = TypedDict(
"MUTABLE_RTW", {"success": bool, "data": Mapping[int, Sequence[bytes]]}
)
@frozen @frozen
class StorageClientMutables: class StorageClientMutables:
""" """
@ -841,6 +1124,29 @@ class StorageClientMutables:
Given a mapping between share numbers and test/write vectors, the tests Given a mapping between share numbers and test/write vectors, the tests
are done and if they are valid the writes are done. are done and if they are valid the writes are done.
""" """
with start_action(
action_type="allmydata:storage:http-client:mutable:read-test-write",
storage_index=si_to_human_readable(storage_index),
):
return await self._read_test_write_chunks(
storage_index,
write_enabler_secret,
lease_renew_secret,
lease_cancel_secret,
testwrite_vectors,
read_vector,
)
async def _read_test_write_chunks(
self,
storage_index: bytes,
write_enabler_secret: bytes,
lease_renew_secret: bytes,
lease_cancel_secret: bytes,
testwrite_vectors: dict[int, TestWriteVectors],
read_vector: list[ReadVector],
) -> ReadTestWriteResult:
"""Implementation of ``read_test_write_chunks()``."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index)) "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index))
) )
@ -860,50 +1166,83 @@ class StorageClientMutables:
message_to_serialize=message, message_to_serialize=message,
) )
if response.code == http.OK: if response.code == http.OK:
result = await self._client.decode_cbor( result = cast(
response, _SCHEMAS["mutable_read_test_write"] MUTABLE_RTW,
await self._client.decode_cbor(
response, _SCHEMAS["mutable_read_test_write"]
),
) )
return ReadTestWriteResult(success=result["success"], reads=result["data"]) return ReadTestWriteResult(success=result["success"], reads=result["data"])
else: else:
raise ClientException(response.code, (await response.content())) raise ClientException(response.code, (await response.content()))
def read_share_chunk( @async_to_deferred
async def read_share_chunk(
self, self,
storage_index: bytes, storage_index: bytes,
share_number: int, share_number: int,
offset: int, offset: int,
length: int, length: int,
) -> Deferred[bytes]: ) -> bytes:
""" """
Download a chunk of data from a share. Download a chunk of data from a share.
""" """
return read_share_chunk( with start_action(
self._client, "mutable", storage_index, share_number, offset, length action_type="allmydata:storage:http-client:mutable:read-share-chunk",
) storage_index=si_to_human_readable(storage_index),
share_number=share_number,
offset=offset,
length=length,
) as ctx:
result = await read_share_chunk(
self._client, "mutable", storage_index, share_number, offset, length
)
ctx.add_success_fields(data_len=len(result))
return result
@async_to_deferred @async_to_deferred
async def list_shares(self, storage_index: bytes) -> set[int]: async def list_shares(self, storage_index: bytes) -> Set[int]:
""" """
List the share numbers for a given storage index. List the share numbers for a given storage index.
""" """
with start_action(
action_type="allmydata:storage:http-client:mutable:list-shares",
storage_index=si_to_human_readable(storage_index),
) as ctx:
result = await self._list_shares(storage_index)
ctx.add_success_fields(shares=result)
return result
async def _list_shares(self, storage_index: bytes) -> Set[int]:
"""Implementation of ``list_shares()``."""
url = self._client.relative_url( url = self._client.relative_url(
"/storage/v1/mutable/{}/shares".format(_encode_si(storage_index)) "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index))
) )
response = await self._client.request("GET", url) response = await self._client.request("GET", url)
if response.code == http.OK: if response.code == http.OK:
return await self._client.decode_cbor( return cast(
response, _SCHEMAS["mutable_list_shares"] Set[int],
await self._client.decode_cbor(
response, _SCHEMAS["mutable_list_shares"]
),
) )
else: else:
raise ClientException(response.code) raise ClientException(response.code)
def advise_corrupt_share( @async_to_deferred
async def advise_corrupt_share(
self, self,
storage_index: bytes, storage_index: bytes,
share_number: int, share_number: int,
reason: str, reason: str,
): ) -> None:
"""Indicate a share has been corrupted, with a human-readable message.""" """Indicate a share has been corrupted, with a human-readable message."""
return advise_corrupt_share( with start_action(
self._client, "mutable", storage_index, share_number, reason action_type="allmydata:storage:http-client:mutable:advise-corrupt-share",
) storage_index=si_to_human_readable(storage_index),
share_number=share_number,
reason=reason,
):
await advise_corrupt_share(
self._client, "mutable", storage_index, share_number, reason
)

17
src/allmydata/storage/http_common.py
View File

@ -12,6 +12,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
from werkzeug.http import parse_options_header from werkzeug.http import parse_options_header
from twisted.web.http_headers import Headers from twisted.web.http_headers import Headers
from twisted.web.iweb import IResponse
CBOR_MIME_TYPE = "application/cbor" CBOR_MIME_TYPE = "application/cbor"
@ -22,13 +23,25 @@ def get_content_type(headers: Headers) -> Optional[str]:
Returns ``None`` if no content-type was set. Returns ``None`` if no content-type was set.
""" """
values = headers.getRawHeaders("content-type") or [None] values = headers.getRawHeaders("content-type", [None]) or [None]
content_type = parse_options_header(values[0])[0] or None content_type = parse_options_header(values[0])[0] or None
return content_type return content_type
def response_is_not_html(response: IResponse) -> None:
"""
During tests, this is registered so we can ensure the web server
doesn't give us text/html.
HTML is never correct except in 404, but it's the default for
Twisted's web server so we assert nothing unexpected happened.
"""
if response.code != 404:
assert get_content_type(response.headers) != "text/html"
def swissnum_auth_header(swissnum: bytes) -> bytes: def swissnum_auth_header(swissnum: bytes) -> bytes:
"""Return value for ``Authentication`` header.""" """Return value for ``Authorization`` header."""
return b"Tahoe-LAFS " + b64encode(swissnum).strip() return b"Tahoe-LAFS " + b64encode(swissnum).strip()

177
src/allmydata/storage/http_server.py
View File

@ -4,7 +4,7 @@ HTTP server for storage.
from __future__ import annotations from __future__ import annotations
from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast from typing import Any, Callable, Union, cast, Optional
from functools import wraps from functools import wraps
from base64 import b64decode from base64 import b64decode
import binascii import binascii
@ -12,6 +12,7 @@ from tempfile import TemporaryFile
from os import SEEK_END, SEEK_SET from os import SEEK_END, SEEK_SET
import mmap import mmap
from eliot import start_action
from cryptography.x509 import Certificate as CryptoCertificate from cryptography.x509 import Certificate as CryptoCertificate
from zope.interface import implementer from zope.interface import implementer
from klein import Klein from klein import Klein
@ -67,14 +68,14 @@ class ClientSecretsException(Exception):
def _extract_secrets( def _extract_secrets(
header_values, required_secrets header_values: list[str], required_secrets: set[Secrets]
): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes] ) -> dict[Secrets, bytes]:
""" """
Given list of values of ``X-Tahoe-Authorization`` headers, and required Given list of values of ``X-Tahoe-Authorization`` headers, and required
secrets, return dictionary mapping secrets to decoded values. secrets, return dictionary mapping secrets to decoded values.
If too few secrets were given, or too many, a ``ClientSecretsException`` is If too few secrets were given, or too many, a ``ClientSecretsException`` is
raised. raised; its text is sent in the HTTP response.
""" """
string_key_to_enum = {e.value: e for e in Secrets} string_key_to_enum = {e.value: e for e in Secrets}
result = {} result = {}
@ -83,6 +84,10 @@ def _extract_secrets(
string_key, string_value = header_value.strip().split(" ", 1) string_key, string_value = header_value.strip().split(" ", 1)
key = string_key_to_enum[string_key] key = string_key_to_enum[string_key]
value = b64decode(string_value) value = b64decode(string_value)
if value == b"":
raise ClientSecretsException(
"Failed to decode secret {}".format(string_key)
)
if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32: if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32:
raise ClientSecretsException("Lease secrets must be 32 bytes long") raise ClientSecretsException("Lease secrets must be 32 bytes long")
result[key] = value result[key] = value
@ -90,37 +95,68 @@ def _extract_secrets(
raise ClientSecretsException("Bad header value(s): {}".format(header_values)) raise ClientSecretsException("Bad header value(s): {}".format(header_values))
if result.keys() != required_secrets: if result.keys() != required_secrets:
raise ClientSecretsException( raise ClientSecretsException(
"Expected {} secrets, got {}".format(required_secrets, result.keys()) "Expected {} in X-Tahoe-Authorization headers, got {}".format(
[r.value for r in required_secrets], list(result.keys())
)
) )
return result return result
def _authorization_decorator(required_secrets): def _authorization_decorator(required_secrets):
""" """
Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization`` 1. Check the ``Authorization`` header matches server swissnum.
headers and pass them in. 2. Extract ``X-Tahoe-Authorization`` headers and pass them in.
3. Log the request and response.
""" """
def decorator(f): def decorator(f):
@wraps(f) @wraps(f)
def route(self, request, *args, **kwargs): def route(self, request, *args, **kwargs):
if not timing_safe_compare( # Don't set text/html content type by default:
request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode( request.defaultContentType = None
"utf-8"
), with start_action(
swissnum_auth_header(self._swissnum), action_type="allmydata:storage:http-server:handle-request",
): method=request.method,
request.setResponseCode(http.UNAUTHORIZED) path=request.path,
return b"" ) as ctx:
authorization = request.requestHeaders.getRawHeaders( try:
"X-Tahoe-Authorization", [] # Check Authorization header:
) try:
try: auth_header = request.requestHeaders.getRawHeaders(
secrets = _extract_secrets(authorization, required_secrets) "Authorization", [""]
except ClientSecretsException: )[0].encode("utf-8")
request.setResponseCode(http.BAD_REQUEST) except UnicodeError:
return b"Missing required secrets" raise _HTTPError(http.BAD_REQUEST, "Bad Authorization header")
return f(self, request, secrets, *args, **kwargs) if not timing_safe_compare(
auth_header,
swissnum_auth_header(self._swissnum),
):
raise _HTTPError(
http.UNAUTHORIZED, "Wrong Authorization header"
)
# Check secrets:
authorization = request.requestHeaders.getRawHeaders(
"X-Tahoe-Authorization", []
)
try:
secrets = _extract_secrets(authorization, required_secrets)
except ClientSecretsException as e:
raise _HTTPError(http.BAD_REQUEST, str(e))
# Run the business logic:
result = f(self, request, secrets, *args, **kwargs)
except _HTTPError as e:
# This isn't an error necessarily for logging purposes,
# it's an implementation detail, an easier way to set
# response codes.
ctx.add_success_fields(response_code=e.code)
ctx.finish()
raise
else:
ctx.add_success_fields(response_code=request.code)
return result
return route return route
@ -173,7 +209,7 @@ class UploadsInProgress(object):
_uploads: dict[bytes, StorageIndexUploads] = Factory(dict) _uploads: dict[bytes, StorageIndexUploads] = Factory(dict)
# Map BucketWriter to (storage index, share number) # Map BucketWriter to (storage index, share number)
_bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict) _bucketwriters: dict[BucketWriter, tuple[bytes, int]] = Factory(dict)
def add_write_bucket( def add_write_bucket(
self, self,
@ -248,8 +284,10 @@ class _HTTPError(Exception):
Raise from ``HTTPServer`` endpoint to return the given HTTP response code. Raise from ``HTTPServer`` endpoint to return the given HTTP response code.
""" """
def __init__(self, code: int): def __init__(self, code: int, body: Optional[str] = None):
Exception.__init__(self, (code, body))
self.code = code self.code = code
self.body = body
# CDDL schemas. # CDDL schemas.
@ -273,7 +311,7 @@ _SCHEMAS = {
"advise_corrupt_share": Schema( "advise_corrupt_share": Schema(
""" """
request = { request = {
reason: tstr reason: tstr .size (1..32765)
} }
""" """
), ),
@ -348,13 +386,16 @@ class _ReadRangeProducer:
a request. a request.
""" """
request: Request request: Optional[Request]
read_data: ReadData read_data: ReadData
result: Deferred result: Optional[Deferred[bytes]]
start: int start: int
remaining: int remaining: int
def resumeProducing(self): def resumeProducing(self):
if self.result is None or self.request is None:
return
to_read = min(self.remaining, 65536) to_read = min(self.remaining, 65536)
data = self.read_data(self.start, to_read) data = self.read_data(self.start, to_read)
assert len(data) <= to_read assert len(data) <= to_read
@ -403,7 +444,7 @@ class _ReadRangeProducer:
def read_range( def read_range(
request: Request, read_data: ReadData, share_length: int request: Request, read_data: ReadData, share_length: int
) -> Union[Deferred, bytes]: ) -> Union[Deferred[bytes], bytes]:
""" """
Read an optional ``Range`` header, reads data appropriately via the given Read an optional ``Range`` header, reads data appropriately via the given
callable, writes the data to the request. callable, writes the data to the request.
@ -440,6 +481,8 @@ def read_range(
raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE) raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)
offset, end = range_header.ranges[0] offset, end = range_header.ranges[0]
assert end is not None # should've exited in block above this if so
# If we're being ask to read beyond the length of the share, just read # If we're being ask to read beyond the length of the share, just read
# less: # less:
end = min(end, share_length) end = min(end, share_length)
@ -458,7 +501,7 @@ def read_range(
ContentRange("bytes", offset, end).to_header(), ContentRange("bytes", offset, end).to_header(),
) )
d = Deferred() d: Deferred[bytes] = Deferred()
request.registerProducer( request.registerProducer(
_ReadRangeProducer( _ReadRangeProducer(
request, read_data_with_error_handling, d, offset, end - offset request, read_data_with_error_handling, d, offset, end - offset
@ -468,6 +511,25 @@ def read_range(
return d return d
def _add_error_handling(app: Klein):
"""Add exception handlers to a Klein app."""
@app.handle_errors(_HTTPError)
def _http_error(_, request, failure):
"""Handle ``_HTTPError`` exceptions."""
request.setResponseCode(failure.value.code)
if failure.value.body is not None:
return failure.value.body
else:
return b""
@app.handle_errors(CDDLValidationError)
def _cddl_validation_error(_, request, failure):
"""Handle CDDL validation errors."""
request.setResponseCode(http.BAD_REQUEST)
return str(failure.value).encode("utf-8")
class HTTPServer(object): class HTTPServer(object):
""" """
A HTTP interface to the storage server. A HTTP interface to the storage server.
@ -475,18 +537,7 @@ class HTTPServer(object):
_app = Klein() _app = Klein()
_app.url_map.converters["storage_index"] = StorageIndexConverter _app.url_map.converters["storage_index"] = StorageIndexConverter
_add_error_handling(_app)
@_app.handle_errors(_HTTPError)
def _http_error(self, request, failure):
"""Handle ``_HTTPError`` exceptions."""
request.setResponseCode(failure.value.code)
return b""
@_app.handle_errors(CDDLValidationError)
def _cddl_validation_error(self, request, failure):
"""Handle CDDL validation errors."""
request.setResponseCode(http.BAD_REQUEST)
return str(failure.value).encode("utf-8")
def __init__( def __init__(
self, self,
@ -592,7 +643,26 @@ class HTTPServer(object):
@_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"]) @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
def version(self, request, authorization): def version(self, request, authorization):
"""Return version information.""" """Return version information."""
return self._send_encoded(request, self._storage_server.get_version()) return self._send_encoded(request, self._get_version())
def _get_version(self) -> dict[bytes, Any]:
"""
Get the HTTP version of the storage server's version response.
This differs from the Foolscap version by omitting certain obsolete
fields.
"""
v = self._storage_server.get_version()
v1_identifier = b"http://allmydata.org/tahoe/protocols/storage/v1"
v1 = v[v1_identifier]
return {
v1_identifier: {
b"maximum-immutable-share-size": v1[b"maximum-immutable-share-size"],
b"maximum-mutable-share-size": v1[b"maximum-mutable-share-size"],
b"available-space": v1[b"available-space"],
},
b"application-version": v[b"application-version"],
}
##### Immutable APIs ##### ##### Immutable APIs #####
@ -731,6 +801,7 @@ class HTTPServer(object):
) )
def read_share_chunk(self, request, authorization, storage_index, share_number): def read_share_chunk(self, request, authorization, storage_index, share_number):
"""Read a chunk for an already uploaded immutable.""" """Read a chunk for an already uploaded immutable."""
request.setHeader("content-type", "application/octet-stream")
try: try:
bucket = self._storage_server.get_buckets(storage_index)[share_number] bucket = self._storage_server.get_buckets(storage_index)[share_number]
except KeyError: except KeyError:
@ -779,7 +850,9 @@ class HTTPServer(object):
# The reason can be a string with explanation, so in theory it could be # The reason can be a string with explanation, so in theory it could be
# longish? # longish?
info = await self._read_encoded( info = await self._read_encoded(
request, _SCHEMAS["advise_corrupt_share"], max_size=32768, request,
_SCHEMAS["advise_corrupt_share"],
max_size=32768,
) )
bucket.advise_corrupt_share(info["reason"].encode("utf-8")) bucket.advise_corrupt_share(info["reason"].encode("utf-8"))
return b"" return b""
@ -834,6 +907,7 @@ class HTTPServer(object):
) )
def read_mutable_chunk(self, request, authorization, storage_index, share_number): def read_mutable_chunk(self, request, authorization, storage_index, share_number):
"""Read a chunk from a mutable.""" """Read a chunk from a mutable."""
request.setHeader("content-type", "application/octet-stream")
try: try:
share_length = self._storage_server.get_mutable_share_length( share_length = self._storage_server.get_mutable_share_length(
@ -926,13 +1000,20 @@ class _TLSEndpointWrapper(object):
def build_nurl( def build_nurl(
hostname: str, port: int, swissnum: str, certificate: CryptoCertificate hostname: str,
port: int,
swissnum: str,
certificate: CryptoCertificate,
subscheme: Optional[str] = None,
) -> DecodedURL: ) -> DecodedURL:
""" """
Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509 Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509
certificate for the server. Clients can then connect to the server using certificate for the server. Clients can then connect to the server using
this NURL. this NURL.
""" """
scheme = "pb"
if subscheme is not None:
scheme = f"{scheme}+{subscheme}"
return DecodedURL().replace( return DecodedURL().replace(
fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap) fragment="v=1", # how we know this NURL is HTTP-based (i.e. not Foolscap)
host=hostname, host=hostname,
@ -944,7 +1025,7 @@ def build_nurl(
"ascii", "ascii",
), ),
), ),
scheme="pb", scheme=scheme,
) )
@ -954,7 +1035,7 @@ def listen_tls(
endpoint: IStreamServerEndpoint, endpoint: IStreamServerEndpoint,
private_key_path: FilePath, private_key_path: FilePath,
cert_path: FilePath, cert_path: FilePath,
) -> Deferred[Tuple[DecodedURL, IListeningPort]]: ) -> Deferred[tuple[DecodedURL, IListeningPort]]:
""" """
Start a HTTPS storage server on the given port, return the NURL and the Start a HTTPS storage server on the given port, return the NURL and the
listening port. listening port.

4
src/allmydata/storage/lease.py
View File

@ -173,7 +173,9 @@ class LeaseInfo(object):
""" """
return attr.assoc( return attr.assoc(
self, self,
_expiration_time=new_expire_time, # MyPy is unhappy with this; long-term solution is likely switch to
# new @frozen attrs API, with type annotations.
_expiration_time=new_expire_time, # type: ignore[call-arg]
) )
def is_renew_secret(self, candidate_secret): def is_renew_secret(self, candidate_secret):

19
src/allmydata/storage/lease_schema.py
View File

@ -2,19 +2,7 @@
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import absolute_import from typing import Union
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from typing import Union
except ImportError:
pass
import attr import attr
@ -68,7 +56,7 @@ class HashedLeaseSerializer(object):
""" """
Hash a lease secret for storage. Hash a lease secret for storage.
""" """
return blake2b(secret, digest_size=32, encoder=RawEncoder()) return blake2b(secret, digest_size=32, encoder=RawEncoder)
@classmethod @classmethod
def _hash_lease_info(cls, lease_info): def _hash_lease_info(cls, lease_info):
@ -95,8 +83,7 @@ class HashedLeaseSerializer(object):
cls._hash_secret, cls._hash_secret,
) )
def serialize(self, lease): def serialize(self, lease: Union[LeaseInfo, HashedLeaseInfo]) -> bytes:
# type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes
if isinstance(lease, LeaseInfo): if isinstance(lease, LeaseInfo):
# v2 of the immutable schema stores lease secrets hashed. If # v2 of the immutable schema stores lease secrets hashed. If
# we're given a LeaseInfo then it holds plaintext secrets. Hash # we're given a LeaseInfo then it holds plaintext secrets. Hash

9
src/allmydata/storage/server.py
View File

@ -2,8 +2,9 @@
Ported to Python 3. Ported to Python 3.
""" """
from __future__ import annotations from __future__ import annotations
from future.utils import bytes_to_native_str from future.utils import bytes_to_native_str
from typing import Dict, Tuple, Iterable from typing import Iterable, Any
import os, re import os, re
@ -54,7 +55,9 @@ class StorageServer(service.MultiService):
""" """
Implement the business logic for the storage server. Implement the business logic for the storage server.
""" """
name = 'storage' # The type in Twisted for services is wrong in 22.10...
# https://github.com/twisted/twisted/issues/10135
name = 'storage' # type: ignore[assignment]
# only the tests change this to anything else # only the tests change this to anything else
LeaseCheckerClass = LeaseCheckingCrawler LeaseCheckerClass = LeaseCheckingCrawler
@ -823,7 +826,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78
self._server = storage_server self._server = storage_server
# Canaries and disconnect markers for BucketWriters created via Foolscap: # Canaries and disconnect markers for BucketWriters created via Foolscap:
self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]] self._bucket_writer_disconnect_markers : dict[BucketWriter, tuple[IRemoteReference, Any]] = {}
self._server.register_bucket_writer_close_handler(self._bucket_writer_closed) self._server.register_bucket_writer_close_handler(self._bucket_writer_closed)

155
src/allmydata/storage_client.py
View File

@ -33,8 +33,7 @@ Ported to Python 3.
from __future__ import annotations from __future__ import annotations
from six import ensure_text from six import ensure_text
from typing import Union, Callable, Any, Optional, cast
from typing import Union, Any
from os import urandom from os import urandom
import re import re
import time import time
@ -45,6 +44,7 @@ import json
import attr import attr
from hyperlink import DecodedURL from hyperlink import DecodedURL
from twisted.web.client import HTTPConnectionPool
from zope.interface import ( from zope.interface import (
Attribute, Attribute,
Interface, Interface,
@ -54,6 +54,7 @@ from twisted.python.failure import Failure
from twisted.web import http from twisted.web import http
from twisted.internet.task import LoopingCall from twisted.internet.task import LoopingCall
from twisted.internet import defer, reactor from twisted.internet import defer, reactor
from twisted.internet.interfaces import IReactorTime
from twisted.application import service from twisted.application import service
from twisted.plugin import ( from twisted.plugin import (
getPlugins, getPlugins,
@ -71,6 +72,7 @@ from allmydata.interfaces import (
IServer, IServer,
IStorageServer, IStorageServer,
IFoolscapStoragePlugin, IFoolscapStoragePlugin,
VersionMessage
) )
from allmydata.grid_manager import ( from allmydata.grid_manager import (
create_grid_manager_verifier, SignedCertificate create_grid_manager_verifier, SignedCertificate
@ -78,17 +80,19 @@ from allmydata.grid_manager import (
from allmydata.crypto import ( from allmydata.crypto import (
ed25519, ed25519,
) )
from allmydata.util.tor_provider import _Provider as TorProvider
from allmydata.util import log, base32, connection_status from allmydata.util import log, base32, connection_status
from allmydata.util.assertutil import precondition from allmydata.util.assertutil import precondition
from allmydata.util.observer import ObserverList from allmydata.util.observer import ObserverList
from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.util.rrefutil import add_version_to_remote_reference
from allmydata.util.hashutil import permute_server_hash from allmydata.util.hashutil import permute_server_hash
from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict
from allmydata.util.deferredutil import async_to_deferred from allmydata.util.deferredutil import async_to_deferred, race
from allmydata.storage.http_client import ( from allmydata.storage.http_client import (
StorageClient, StorageClientImmutables, StorageClientGeneral, StorageClient, StorageClientImmutables, StorageClientGeneral,
ClientException as HTTPClientException, StorageClientMutables, ClientException as HTTPClientException, StorageClientMutables,
ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException,
StorageClientFactory
) )
from .node import _Config from .node import _Config
@ -203,8 +207,13 @@ class StorageFarmBroker(service.MultiService):
tub_maker, tub_maker,
node_config: _Config, node_config: _Config,
storage_client_config=None, storage_client_config=None,
default_connection_handlers=None,
tor_provider: Optional[TorProvider]=None,
): ):
service.MultiService.__init__(self) service.MultiService.__init__(self)
if default_connection_handlers is None:
default_connection_handlers = {"tcp": "tcp"}
assert permute_peers # False not implemented yet assert permute_peers # False not implemented yet
self.permute_peers = permute_peers self.permute_peers = permute_peers
self._tub_maker = tub_maker self._tub_maker = tub_maker
@ -224,6 +233,8 @@ class StorageFarmBroker(service.MultiService):
self.introducer_client = None self.introducer_client = None
self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred) self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred)
self._connected_high_water_mark = 0 self._connected_high_water_mark = 0
self._tor_provider = tor_provider
self._default_connection_handlers = default_connection_handlers
@log_call(action_type=u"storage-client:broker:set-static-servers") @log_call(action_type=u"storage-client:broker:set-static-servers")
def set_static_servers(self, servers): def set_static_servers(self, servers):
@ -316,6 +327,8 @@ class StorageFarmBroker(service.MultiService):
server_id, server_id,
server["ann"], server["ann"],
grid_manager_verifier=gm_verifier, grid_manager_verifier=gm_verifier,
default_connection_handlers=self._default_connection_handlers,
tor_provider=self._tor_provider
) )
s.on_status_changed(lambda _: self._got_connection()) s.on_status_changed(lambda _: self._got_connection())
return s return s
@ -1020,6 +1033,26 @@ class NativeStorageServer(service.MultiService):
self._reconnector.reset() self._reconnector.reset()
@async_to_deferred
async def _pick_a_http_server(
reactor,
nurls: list[DecodedURL],
request: Callable[[Any, DecodedURL], defer.Deferred[Any]]
) -> DecodedURL:
"""Pick the first server we successfully send a request to.
Fires with ``None`` if no server was found, or with the ``DecodedURL`` of
the first successfully-connected server.
"""
queries = race([
request(reactor, nurl).addCallback(lambda _, nurl=nurl: nurl)
for nurl in nurls
])
_, nurl = await queries
return nurl
@implementer(IServer) @implementer(IServer)
class HTTPNativeStorageServer(service.MultiService): class HTTPNativeStorageServer(service.MultiService):
""" """
@ -1030,7 +1063,7 @@ class HTTPNativeStorageServer(service.MultiService):
"connected". "connected".
""" """
def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None): def __init__(self, server_id: bytes, announcement, default_connection_handlers: dict[str,str], reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None):
service.MultiService.__init__(self) service.MultiService.__init__(self)
assert isinstance(server_id, bytes) assert isinstance(server_id, bytes)
self._server_id = server_id self._server_id = server_id
@ -1038,6 +1071,10 @@ class HTTPNativeStorageServer(service.MultiService):
self._on_status_changed = ObserverList() self._on_status_changed = ObserverList()
self._reactor = reactor self._reactor = reactor
self._grid_manager_verifier = grid_manager_verifier self._grid_manager_verifier = grid_manager_verifier
self._storage_client_factory = StorageClientFactory(
default_connection_handlers, tor_provider
)
furl = announcement["anonymous-storage-FURL"].encode("utf-8") furl = announcement["anonymous-storage-FURL"].encode("utf-8")
( (
self._nickname, self._nickname,
@ -1046,17 +1083,16 @@ class HTTPNativeStorageServer(service.MultiService):
self._short_description, self._short_description,
self._long_description self._long_description
) = _parse_announcement(server_id, furl, announcement) ) = _parse_announcement(server_id, furl, announcement)
# TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs? self._nurls = [
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935 DecodedURL.from_text(u)
nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0]) for u in announcement[ANONYMOUS_STORAGE_NURLS]
self._istorage_server = _HTTPStorageServer.from_http_client( ]
StorageClient.from_nurl(nurl, reactor) self._istorage_server : Optional[_HTTPStorageServer] = None
)
self._connection_status = connection_status.ConnectionStatus.unstarted() self._connection_status = connection_status.ConnectionStatus.unstarted()
self._version = None self._version = None
self._last_connect_time = None self._last_connect_time = None
self._connecting_deferred = None self._connecting_deferred : Optional[defer.Deferred[object]]= None
def get_permutation_seed(self): def get_permutation_seed(self):
return self._permutation_seed return self._permutation_seed
@ -1168,19 +1204,85 @@ class HTTPNativeStorageServer(service.MultiService):
def try_to_connect(self): def try_to_connect(self):
self._connect() self._connect()
def _connect(self): def _connect(self) -> defer.Deferred[object]:
result = self._istorage_server.get_version() """
Try to connect to a working storage server.
def remove_connecting_deferred(result): If called while a previous ``_connect()`` is already running, it will
just return the same ``Deferred``.
``LoopingCall.stop()`` doesn't cancel ``Deferred``s, unfortunately:
https://github.com/twisted/twisted/issues/11814. Thus we want to store
the ``Deferred`` so we can cancel it when necessary.
We also want to return it so that loop iterations take it into account,
and a new iteration doesn't start while we're in the middle of the
previous one.
"""
# Conceivably try_to_connect() was called on this before, in which case
# we already are in the middle of connecting. So in that case just
# return whatever is in progress:
if self._connecting_deferred is not None:
return self._connecting_deferred
def done(_):
self._connecting_deferred = None self._connecting_deferred = None
return result
connecting = self._pick_server_and_get_version()
# Set a short timeout since we're relying on this for server liveness. # Set a short timeout since we're relying on this for server liveness.
self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth( connecting = connecting.addTimeout(5, self._reactor).addCallbacks(
remove_connecting_deferred).addCallbacks( self._got_version, self._failed_to_connect
self._got_version, ).addBoth(done)
self._failed_to_connect self._connecting_deferred = connecting
) return connecting
@async_to_deferred
async def _pick_server_and_get_version(self):
"""
Minimal implementation of connection logic: pick a server, get its
version. This doesn't deal with errors much, so as to minimize
statefulness. It does change ``self._istorage_server``, so possibly
more refactoring would be useful to remove even that much statefulness.
"""
async def get_istorage_server() -> _HTTPStorageServer:
if self._istorage_server is not None:
return self._istorage_server
# We haven't selected a server yet, so let's do so.
# TODO This is somewhat inefficient on startup: it takes two successful
# version() calls before we are live talking to a server, it could only
# be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992
@async_to_deferred
async def request(reactor, nurl: DecodedURL):
# Since we're just using this one off to check if the NURL
# works, no need for persistent pool or other fanciness.
pool = HTTPConnectionPool(reactor, persistent=False)
pool.retryAutomatically = False
storage_client = await self._storage_client_factory.create_storage_client(
nurl, reactor, pool
)
return await StorageClientGeneral(storage_client).get_version()
nurl = await _pick_a_http_server(reactor, self._nurls, request)
# If we've gotten this far, we've found a working NURL.
storage_client = await self._storage_client_factory.create_storage_client(
nurl, cast(IReactorTime, reactor), None
)
self._istorage_server = _HTTPStorageServer.from_http_client(storage_client)
return self._istorage_server
try:
storage_server = await get_istorage_server()
# Get the version from the remote server.
version = await storage_server.get_version()
return version
except Exception as e:
log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
raise
def stopService(self): def stopService(self):
if self._connecting_deferred is not None: if self._connecting_deferred is not None:
@ -1190,6 +1292,11 @@ class HTTPNativeStorageServer(service.MultiService):
if self._lc.running: if self._lc.running:
self._lc.stop() self._lc.stop()
self._failed_to_connect("shut down") self._failed_to_connect("shut down")
if self._istorage_server is not None:
client_shutting_down = self._istorage_server._http_client.shutdown()
result.addCallback(lambda _: client_shutting_down)
return result return result
@ -1355,7 +1462,7 @@ class _HTTPBucketWriter(object):
return self.finished return self.finished
def _ignore_404(failure: Failure) -> Union[Failure, None]: def _ignore_404(failure: Failure) -> Optional[Failure]:
""" """
Useful for advise_corrupt_share(), since it swallows unknown share numbers Useful for advise_corrupt_share(), since it swallows unknown share numbers
in Foolscap. in Foolscap.
@ -1397,13 +1504,13 @@ class _HTTPStorageServer(object):
_http_client = attr.ib(type=StorageClient) _http_client = attr.ib(type=StorageClient)
@staticmethod @staticmethod
def from_http_client(http_client): # type: (StorageClient) -> _HTTPStorageServer def from_http_client(http_client: StorageClient) -> _HTTPStorageServer:
""" """
Create an ``IStorageServer`` from a HTTP ``StorageClient``. Create an ``IStorageServer`` from a HTTP ``StorageClient``.
""" """
return _HTTPStorageServer(http_client=http_client) return _HTTPStorageServer(http_client=http_client)
def get_version(self): def get_version(self) -> defer.Deferred[VersionMessage]:
return StorageClientGeneral(self._http_client).get_version() return StorageClientGeneral(self._http_client).get_version()
@defer.inlineCallbacks @defer.inlineCallbacks

Some files were not shown because too many files have changed in this diff Show More