ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-24 07:06:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 3957-mutable-over-http-speed

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2023-01-23 11:39:54 -05:00
commit 3e6778268f
50 changed files with 19450 additions and 633 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

153
.circleci/config.yml
View File

@ -11,20 +11,60 @@
#
version: 2.1
# A template that can be shared between the two different image-building
# workflows.
.images: &IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub credentials
# available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-11": &DOCKERHUB_CONTEXT
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
parameters:
# Control whether the image-building workflow runs as part of this pipeline.
# Generally we do not want this to run because we don't need our
# dependencies to move around all the time and because building the image
# takes a couple minutes.
#
# An easy way to trigger a pipeline with this set to true is with the
# rebuild-images.sh tool in this directory. You can also do so via the
# CircleCI web UI.
build-images:
default: false
type: "boolean"
# Control whether the test-running workflow runs as part of this pipeline.
# Generally we do want this to run because running the tests is the primary
# purpose of this pipeline.
run-tests:
default: true
type: "boolean"
workflows:
ci:
when: "<< pipeline.parameters.run-tests >>"
jobs:
# Start with jobs testing various platforms.
- "debian-10":
{}
- "debian-11":
{}
- "ubuntu-20-04":
{}
- "ubuntu-18-04":
requires:
- "ubuntu-20-04"
# Equivalent to RHEL 8; CentOS 8 is dead.
- "oraclelinux-8":
@ -54,6 +94,9 @@ workflows:
{}
- "integration":
# Run even the slow integration tests here. We need the `--` to
# sneak past tox and get to pytest.
tox-args: "-- --runslow integration"
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
@ -65,41 +108,10 @@ workflows:
{}
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
# faster and takes various spurious failures out of the critical path.
triggers:
# Build once a day
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- "master"
<<: *IMAGES
jobs:
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub
# credentials available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-10": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "build-image-debian-11":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
- "build-image-oraclelinux-8":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
# Build as part of the workflow but only if requested.
when: "<< pipeline.parameters.build-images >>"
jobs:
@ -167,12 +179,7 @@ jobs:
command: |
dist/Tahoe-LAFS/tahoe --version
debian-10: &DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:10-py3.7"
user: "nobody"
debian-11: &DEBIAN
environment: &UTF_8_ENVIRONMENT
# In general, the test suite is not allowed to fail while the job
# succeeds. But you can set this to "yes" if you want it to be
@ -184,7 +191,7 @@ jobs:
# filenames and argv).
LANG: "en_US.UTF-8"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
@ -252,15 +259,11 @@ jobs:
/tmp/venv/bin/codecov
fi
debian-11:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Restore later using PyPy3.8
# pypy27-buster:
@ -294,6 +297,14 @@ jobs:
integration:
<<: *DEBIAN
parameters:
tox-args:
description: >-
Additional arguments to pass to the tox command.
type: "string"
default: ""
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
@ -306,28 +317,15 @@ jobs:
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
# Pass on anything we got in our parameters.
TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >>"
steps:
- "checkout"
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
ubuntu-18-04: &UBUNTU_18_04
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# The default trial args include --rterrors which is incompatible with
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
ubuntu-20-04:
<<: *DEBIAN
docker:
@ -445,7 +443,7 @@ jobs:
typechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -457,7 +455,7 @@ jobs:
docs:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3.7"
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
@ -508,15 +506,6 @@ jobs:
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
build-image-debian-10:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "10"
PYTHON_VERSION: "3.7"
build-image-debian-11:
<<: *BUILD_IMAGE
@ -525,14 +514,6 @@ jobs:
TAG: "11"
PYTHON_VERSION: "3.9"
build-image-ubuntu-18-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
PYTHON_VERSION: "3.7"
build-image-ubuntu-20-04:
<<: *BUILD_IMAGE

20
.circleci/rebuild-images.sh Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
# Get your API token here:
# https://app.circleci.com/settings/user/tokens
API_TOKEN=$1
shift
# Name the branch you want to trigger the build for
BRANCH=$1
shift
curl \
--verbose \
--request POST \
--url https://circleci.com/api/v2/project/gh/tahoe-lafs/tahoe-lafs/pipeline \
--header "Circle-Token: $API_TOKEN" \
--header "content-type: application/json" \
--data '{"branch":"'"$BRANCH"'","parameters":{"build-images":true,"run-tests":false}}'

9
.circleci/run-tests.sh
View File

@ -45,14 +45,15 @@ fi
# A prefix for the test command that ensure it will exit after no more than a
# certain amount of time. Ideally, we would only enforce a "silent" period
# timeout but there isn't obviously a ready-made tool for that. The test
# suite only takes about 5 - 6 minutes on CircleCI right now. 15 minutes
# seems like a moderately safe window.
# timeout but there isn't obviously a ready-made tool for that. The unit test
# suite only takes about 5 - 6 minutes on CircleCI right now. The integration
# tests are a bit longer than that. 45 minutes seems like a moderately safe
# window.
#
# This is primarily aimed at catching hangs on the PyPy job which runs for
# about 21 minutes and then gets killed by CircleCI in a way that fails the
# job and bypasses our "allowed failure" logic.
TIMEOUT="timeout --kill-after 1m 25m"
TIMEOUT="timeout --kill-after 1m 45m"
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable

22
.github/workflows/ci.yml vendored
View File

@ -48,21 +48,20 @@ jobs:
- windows-latest
- ubuntu-latest
python-version:
- "3.7"
- "3.8"
- "3.9"
- "3.10"
include:
# On macOS don't bother with 3.7-3.8, just to get faster builds.
# On macOS don't bother with 3.8, just to get faster builds.
- os: macos-latest
python-version: "3.9"
- os: macos-latest
python-version: "3.10"
# We only support PyPy on Linux at the moment.
- os: ubuntu-latest
python-version: "pypy-3.7"
- os: ubuntu-latest
python-version: "pypy-3.8"
- os: ubuntu-latest
python-version: "pypy-3.9"
steps:
# See https://github.com/actions/checkout. A fetch-depth of 0
@ -87,8 +86,20 @@ jobs:
run: python misc/build_helpers/show-tool-versions.py
- name: Run tox for corresponding Python version
if: ${{ !contains(matrix.os, 'windows') }}
run: python -m tox
# On Windows, a non-blocking pipe might respond (when emulating Unix-y
# API) with ENOSPC to indicate buffer full. Trial doesn't handle this
# well, so it breaks test runs. To attempt to solve this, we pipe the
# output through passthrough.py that will hopefully be able to do the right
# thing by using Windows APIs.
- name: Run tox for corresponding Python version
if: ${{ contains(matrix.os, 'windows') }}
run: |
pip install twisted pywin32
python -m tox | python misc/windows-enospc/passthrough.py
- name: Upload eliot.log
uses: actions/upload-artifact@v3
with:
@ -162,9 +173,6 @@ jobs:
force-foolscap: false
# 22.04 has some issue with Tor at the moment:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
- os: ubuntu-20.04
python-version: "3.7"
force-foolscap: true
- os: ubuntu-20.04
python-version: "3.9"
force-foolscap: false

2
README.rst
View File

@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS <docs/running.rst>`__
🐍 Python 2
-----------
Python 3.7 or later is now required.
Python 3.8 or later is required.
If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1.

2
default.nix
View File

@ -29,7 +29,7 @@ in
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
# for dependency resolution
, pythonVersion ? "python37" # a string choosing the python derivation from
, pythonVersion ? "python39" # a string choosing the python derivation from
# nixpkgs to target
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,

22
integration/conftest.py
View File

@ -40,7 +40,6 @@ from .util import (
await_client_ready,
TahoeProcess,
cli,
_run_node,
generate_ssh_key,
block_with_timeout,
)
@ -63,6 +62,22 @@ def pytest_addoption(parser):
help=("If set, force Foolscap only for the storage protocol. " +
"Otherwise HTTP will be used.")
)
parser.addoption(
"--runslow", action="store_true", default=False,
dest="runslow",
help="If set, run tests marked as slow.",
)
def pytest_collection_modifyitems(session, config, items):
if not config.option.runslow:
# The --runslow option was not given; keep only collected items not
# marked as slow.
items[:] = [
item
for item
in items
if item.get_closest_marker("slow") is None
]
@pytest.fixture(autouse=True, scope='session')
@ -408,10 +423,9 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
pytest_twisted.blockon(process.restart_async(reactor, request))
await_client_ready(process)
print(f"Alice pid: {process.transport.pid}")
return process

119
integration/test_vectors.py Normal file
View File

@ -0,0 +1,119 @@
"""
Verify certain results against test vectors with well-known results.
"""
from __future__ import annotations
from functools import partial
from typing import AsyncGenerator, Iterator
from itertools import starmap, product
from attrs import evolve
from pytest import mark
from pytest_twisted import ensureDeferred
from . import vectors
from .vectors import parameters
from .util import reconfigure, upload, TahoeProcess
@mark.parametrize('convergence', parameters.CONVERGENCE_SECRETS)
def test_convergence(convergence):
"""
Convergence secrets are 16 bytes.
"""
assert isinstance(convergence, bytes), "Convergence secret must be bytes"
assert len(convergence) == 16, "Convergence secret must by 16 bytes"
@mark.slow
@mark.parametrize('case,expected', vectors.capabilities.items())
@ensureDeferred
async def test_capability(reactor, request, alice, case, expected):
"""
The capability that results from uploading certain well-known data
with certain well-known parameters results in exactly the previously
computed value.
"""
# rewrite alice's config to match params and convergence
await reconfigure(reactor, request, alice, (1, case.params.required, case.params.total), case.convergence)
# upload data in the correct format
actual = upload(alice, case.fmt, case.data)
# compare the resulting cap to the expected result
assert actual == expected
@ensureDeferred
async def skiptest_generate(reactor, request, alice):
"""
This is a helper for generating the test vectors.
You can re-generate the test vectors by fixing the name of the test and
running it. Normally this test doesn't run because it ran once and we
captured its output. Other tests run against that output and we want them
to run against the results produced originally, not a possibly
ever-changing set of outputs.
"""
space = starmap(
# segment_size could be a parameter someday but it's not easy to vary
# using the Python implementation so it isn't one for now.
partial(vectors.Case, segment_size=parameters.SEGMENT_SIZE),
product(
parameters.ZFEC_PARAMS,
parameters.CONVERGENCE_SECRETS,
parameters.OBJECT_DESCRIPTIONS,
parameters.FORMATS,
),
)
iterresults = generate(reactor, request, alice, space)
results = []
async for result in iterresults:
# Accumulate the new result
results.append(result)
# Then rewrite the whole output file with the new accumulator value.
# This means that if we fail partway through, we will still have
# recorded partial results -- instead of losing them all.
vectors.save_capabilities(results)
async def generate(
reactor,
request,
alice: TahoeProcess,
cases: Iterator[vectors.Case],
) -> AsyncGenerator[[vectors.Case, str], None]:
"""
Generate all of the test vectors using the given node.
:param reactor: The reactor to use to restart the Tahoe-LAFS node when it
needs to be reconfigured.
:param request: The pytest request object to use to arrange process
cleanup.
:param format: The name of the encryption/data format to use.
:param alice: The Tahoe-LAFS node to use to generate the test vectors.
:param case: The inputs for which to generate a value.
:return: The capability for the case.
"""
# Share placement doesn't affect the resulting capability. For maximum
# reliability of this generator, be happy if we can put shares anywhere
happy = 1
for case in cases:
await reconfigure(
reactor,
request,
alice,
(happy, case.params.required, case.params.total),
case.convergence
)
# Give the format a chance to make an RSA key if it needs it.
case = evolve(case, fmt=case.fmt.customize())
cap = upload(alice, case.fmt, case.data)
yield case, cap

24
integration/test_web.py
View File

@ -7,18 +7,9 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did
at the time of testing' -- not necessarily a cohesive idea of what the
WebAPI *should* do in every situation. It's not clear the latter
exists anywhere, however.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import time
from urllib.parse import unquote as url_unquote, quote as url_quote
@ -32,6 +23,7 @@ import requests
import html5lib
from bs4 import BeautifulSoup
from pytest_twisted import ensureDeferred
def test_index(alice):
"""
@ -252,10 +244,18 @@ def test_status(alice):
assert found_download, "Failed to find the file we downloaded in the status-page"
def test_directory_deep_check(alice):
@ensureDeferred
async def test_directory_deep_check(reactor, request, alice):
"""
use deep-check and confirm the result pages work
"""
# Make sure the node is configured compatibly with expectations of this
# test.
happy = 3
required = 2
total = 4
await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
# create a directory
resp = requests.post(
@ -313,7 +313,7 @@ def test_directory_deep_check(alice):
)
def check_repair_data(checkdata):
assert checkdata["healthy"] is True
assert checkdata["healthy"]
assert checkdata["count-happiness"] == 4
assert checkdata["count-good-share-hosts"] == 4
assert checkdata["count-shares-good"] == 4

267
integration/util.py
View File

@ -1,14 +1,19 @@
"""
Ported to Python 3.
General functionality useful for the implementation of integration tests.
"""
from __future__ import annotations
from contextlib import contextmanager
from typing import Any
from typing_extensions import Literal
from tempfile import NamedTemporaryFile
import sys
import time
import json
from os import mkdir, environ
from os.path import exists, join
from io import StringIO, BytesIO
from functools import partial
from subprocess import check_output
from twisted.python.filepath import (
@ -18,12 +23,23 @@ from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
from twisted.internet.threads import deferToThread
from twisted.internet.interfaces import IProcessTransport, IReactorProcess
from attrs import frozen, evolve
import requests
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.serialization import (
Encoding,
PrivateFormat,
NoEncryption,
)
from paramiko.rsakey import RSAKey
from boltons.funcutils import wraps
from allmydata.util import base32
from allmydata.util.configutil import (
get_config,
set_config,
@ -134,9 +150,40 @@ class _MagicTextProtocol(ProcessProtocol):
sys.stdout.write(data)
def _cleanup_tahoe_process(tahoe_transport, exited):
def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
"""
Terminate the given process with a kill signal (SIGKILL on POSIX,
If the given process transport seems to still be associated with a
running process, send a SIGTERM to that process.
:param transport: The transport to use.
:param allow_missing: If ``True`` then it is not an error for the
transport to have no associated process. Otherwise, an exception will
be raised in that case.
:raise: ``ValueError`` if ``allow_missing`` is ``False`` and the transport
has no process.
"""
if transport.pid is None:
if allow_missing:
print("Process already cleaned up and that's okay.")
return
else:
raise ValueError("Process is not running")
print("signaling {} with TERM".format(transport.pid))
try:
transport.signalProcess('TERM')
except ProcessExitedAlready:
# The transport object thought it still had a process but the real OS
# process has already exited. That's fine. We accomplished what we
# wanted to. We don't care about ``allow_missing`` here because
# there's no way we could have known the real OS process already
# exited.
pass
def _cleanup_tahoe_process(tahoe_transport, exited, allow_missing=False):
"""
Terminate the given process with a kill signal (SIGTERM on POSIX,
TerminateProcess on Windows).
:param tahoe_transport: The `IProcessTransport` representing the process.
@ -145,14 +192,10 @@ def _cleanup_tahoe_process(tahoe_transport, exited):
:return: After the process has exited.
"""
from twisted.internet import reactor
try:
print("signaling {} with TERM".format(tahoe_transport.pid))
tahoe_transport.signalProcess('TERM')
print("signaled, blocking on exit")
block_with_timeout(exited, reactor)
print("exited, goodbye")
except ProcessExitedAlready:
pass
_cleanup_process_async(tahoe_transport, allow_missing=allow_missing)
print("signaled, blocking on exit")
block_with_timeout(exited, reactor)
print("exited, goodbye")
def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
@ -199,8 +242,33 @@ class TahoeProcess(object):
def kill(self):
"""Kill the process, block until it's done."""
print(f"TahoeProcess.kill({self.transport.pid} / {self.node_dir})")
_cleanup_tahoe_process(self.transport, self.transport.exited)
def kill_async(self):
"""
Kill the process, return a Deferred that fires when it's done.
"""
print(f"TahoeProcess.kill_async({self.transport.pid} / {self.node_dir})")
_cleanup_process_async(self.transport, allow_missing=False)
return self.transport.exited
def restart_async(self, reactor: IReactorProcess, request: Any) -> Deferred:
"""
Stop and then re-start the associated process.
:return: A Deferred that fires after the new process is ready to
handle requests.
"""
d = self.kill_async()
d.addCallback(lambda ignored: _run_node(reactor, self.node_dir, request, None, finalize=False))
def got_new_process(proc):
# Grab the new transport since the one we had before is no longer
# valid after the stop/start cycle.
self._process_transport = proc.transport
d.addCallback(got_new_process)
return d
def __str__(self):
return "<TahoeProcess in '{}'>".format(self._node_dir)
@ -229,19 +297,17 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
)
transport.exited = protocol.exited
tahoe_process = TahoeProcess(
transport,
node_dir,
)
if finalize:
request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
request.addfinalizer(tahoe_process.kill)
# XXX abusing the Deferred; should use .when_magic_seen() pattern
def got_proto(proto):
transport._protocol = proto
return TahoeProcess(
transport,
node_dir,
)
protocol.magic_seen.addCallback(got_proto)
return protocol.magic_seen
d = protocol.magic_seen
d.addCallback(lambda ignored: tahoe_process)
return d
def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
@ -572,3 +638,158 @@ def run_in_thread(f):
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test
@frozen
class CHK:
"""
Represent the CHK encoding sufficiently to run a ``tahoe put`` command
using it.
"""
kind = "chk"
max_shares = 256
def customize(self) -> CHK:
# Nothing to do.
return self
@classmethod
def load(cls, params: None) -> CHK:
assert params is None
return cls()
def to_json(self) -> None:
return None
@contextmanager
def to_argv(self) -> None:
yield []
@frozen
class SSK:
"""
Represent the SSK encodings (SDMF and MDMF) sufficiently to run a
``tahoe put`` command using one of them.
"""
kind = "ssk"
# SDMF and MDMF encode share counts (N and k) into the share itself as an
# unsigned byte. They could have encoded (share count - 1) to fit the
# full range supported by ZFEC into the unsigned byte - but they don't.
# So 256 is inaccessible to those formats and we set the upper bound at
# 255.
max_shares = 255
name: Literal["sdmf", "mdmf"]
key: None | bytes
@classmethod
def load(cls, params: dict) -> SSK:
assert params.keys() == {"format", "mutable", "key"}
return cls(params["format"], params["key"].encode("ascii"))
def customize(self) -> SSK:
"""
Return an SSK with a newly generated random RSA key.
"""
return evolve(self, key=generate_rsa_key())
def to_json(self) -> dict[str, str]:
return {
"format": self.name,
"mutable": None,
"key": self.key.decode("ascii"),
}
@contextmanager
def to_argv(self) -> None:
with NamedTemporaryFile() as f:
f.write(self.key)
f.flush()
yield [f"--format={self.name}", "--mutable", f"--private-key-path={f.name}"]
def upload(alice: TahoeProcess, fmt: CHK | SSK, data: bytes) -> str:
"""
Upload the given data to the given node.
:param alice: The node to upload to.
:param fmt: The name of the format for the upload. CHK, SDMF, or MDMF.
:param data: The data to upload.
:return: The capability for the uploaded data.
"""
with NamedTemporaryFile() as f:
f.write(data)
f.flush()
with fmt.to_argv() as fmt_argv:
argv = [alice, "put"] + fmt_argv + [f.name]
return cli(*argv).decode("utf-8").strip()
async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int, int], convergence: None | bytes) -> None:
"""
Reconfigure a Tahoe-LAFS node with different ZFEC parameters and
convergence secret.
If the current configuration is different from the specified
configuration, the node will be restarted so it takes effect.
:param reactor: A reactor to use to restart the process.
:param request: The pytest request object to use to arrange process
cleanup.
:param node: The Tahoe-LAFS node to reconfigure.
:param params: The ``happy``, ``needed``, and ``total`` ZFEC encoding
parameters.
:param convergence: If given, the convergence secret. If not given, the
existing convergence secret will be left alone.
:return: ``None`` after the node configuration has been rewritten, the
node has been restarted, and the node is ready to provide service.
"""
happy, needed, total = params
config = node.get_config()
changed = False
cur_happy = int(config.get_config("client", "shares.happy"))
cur_needed = int(config.get_config("client", "shares.needed"))
cur_total = int(config.get_config("client", "shares.total"))
if (happy, needed, total) != (cur_happy, cur_needed, cur_total):
changed = True
config.set_config("client", "shares.happy", str(happy))
config.set_config("client", "shares.needed", str(needed))
config.set_config("client", "shares.total", str(total))
if convergence is not None:
cur_convergence = config.get_private_config("convergence").encode("ascii")
if base32.a2b(cur_convergence) != convergence:
changed = True
config.write_private_config("convergence", base32.b2a(convergence))
if changed:
# restart the node
print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
await node.restart_async(reactor, request)
print("Restarted. Waiting for ready state.")
await_client_ready(node)
print("Ready.")
else:
print("Config unchanged, not restarting.")
def generate_rsa_key() -> bytes:
"""
Generate a 2048 bit RSA key suitable for use with SSKs.
"""
return rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
).private_bytes(
encoding=Encoding.PEM,
format=PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=NoEncryption(),
)

30
integration/vectors/__init__.py Normal file
View File

@ -0,0 +1,30 @@
__all__ = [
"DATA_PATH",
"CURRENT_VERSION",
"MAX_SHARES",
"Case",
"Sample",
"SeedParam",
"encode_bytes",
"save_capabilities",
"capabilities",
]
from .vectors import (
DATA_PATH,
CURRENT_VERSION,
Case,
Sample,
SeedParam,
encode_bytes,
save_capabilities,
capabilities,
)
from .parameters import (
MAX_SHARES,
)

58
integration/vectors/model.py Normal file
View File

@ -0,0 +1,58 @@
"""
Simple data type definitions useful in the definition/verification of test
vectors.
"""
from __future__ import annotations
from attrs import frozen
# CHK have a max of 256 shares. SDMF / MDMF have a max of 255 shares!
# Represent max symbolically and resolve it when we know what format we're
# dealing with.
MAX_SHARES = "max"
@frozen
class Sample:
"""
Some instructions for building a long byte string.
:ivar seed: Some bytes to repeat some times to produce the string.
:ivar length: The length of the desired byte string.
"""
seed: bytes
length: int
@frozen
class Param:
"""
Some ZFEC parameters.
"""
required: int
total: int
@frozen
class SeedParam:
"""
Some ZFEC parameters, almost.
:ivar required: The number of required shares.
:ivar total: Either the number of total shares or the constant
``MAX_SHARES`` to indicate that the total number of shares should be
the maximum number supported by the object format.
"""
required: int
total: int | str
def realize(self, max_total: int) -> Param:
"""
Create a ``Param`` from this object's values, possibly
substituting the given real value for total if necessary.
:param max_total: The value to use to replace ``MAX_SHARES`` if
necessary.
"""
if self.total == MAX_SHARES:
return Param(self.required, max_total)
return Param(self.required, self.total)

93
integration/vectors/parameters.py Normal file
View File

@ -0,0 +1,93 @@
"""
Define input parameters for test vector generation.
:ivar CONVERGENCE_SECRETS: Convergence secrets.
:ivar SEGMENT_SIZE: The single segment size that the Python implementation
currently supports without a lot of refactoring.
:ivar OBJECT_DESCRIPTIONS: Small objects with instructions which can be
expanded into a possibly large byte string. These are intended to be used
as plaintext inputs.
:ivar ZFEC_PARAMS: Input parameters to ZFEC.
:ivar FORMATS: Encoding/encryption formats.
"""
from __future__ import annotations
from hashlib import sha256
from .model import MAX_SHARES
from .vectors import Sample, SeedParam
from ..util import CHK, SSK
def digest(bs: bytes) -> bytes:
"""
Digest bytes to bytes.
"""
return sha256(bs).digest()
def hexdigest(bs: bytes) -> str:
"""
Digest bytes to text.
"""
return sha256(bs).hexdigest()
# Just a couple convergence secrets. The only thing we do with this value is
# feed it into a tagged hash. It certainly makes a difference to the output
# but the hash should destroy any structure in the input so it doesn't seem
# like there's a reason to test a lot of different values.
CONVERGENCE_SECRETS: list[bytes] = [
b"aaaaaaaaaaaaaaaa",
digest(b"Hello world")[:16],
]
SEGMENT_SIZE: int = 128 * 1024
# Exercise at least a handful of different sizes, trying to cover:
#
# 1. Some cases smaller than one "segment" (128k).
# This covers shrinking of some parameters to match data size.
# This includes one case of the smallest possible CHK.
#
# 2. Some cases right on the edges of integer segment multiples.
# Because boundaries are tricky.
#
# 4. Some cases that involve quite a few segments.
# This exercises merkle tree construction more thoroughly.
#
# See ``stretch`` for construction of the actual test data.
OBJECT_DESCRIPTIONS: list[Sample] = [
# The smallest possible. 55 bytes and smaller are LIT.
Sample(b"a", 56),
Sample(b"a", 1024),
Sample(b"c", 4096),
Sample(digest(b"foo"), SEGMENT_SIZE - 1),
Sample(digest(b"bar"), SEGMENT_SIZE + 1),
Sample(digest(b"baz"), SEGMENT_SIZE * 16 - 1),
Sample(digest(b"quux"), SEGMENT_SIZE * 16 + 1),
Sample(digest(b"bazquux"), SEGMENT_SIZE * 32),
Sample(digest(b"foobar"), SEGMENT_SIZE * 64 - 1),
Sample(digest(b"barbaz"), SEGMENT_SIZE * 64 + 1),
]
ZFEC_PARAMS: list[SeedParam] = [
SeedParam(1, 1),
SeedParam(1, 3),
SeedParam(2, 3),
SeedParam(3, 10),
SeedParam(71, 255),
SeedParam(101, MAX_SHARES),
]
FORMATS: list[CHK | SSK] = [
CHK(),
# These start out unaware of a key but various keys will be supplied
# during generation.
SSK(name="sdmf", key=None),
SSK(name="mdmf", key=None),
]

18002
integration/vectors/test_vectors.yaml Executable file
View File

File diff suppressed because it is too large Load Diff

155
integration/vectors/vectors.py Normal file
View File

@ -0,0 +1,155 @@
"""
A module that loads pre-generated test vectors.
:ivar DATA_PATH: The path of the file containing test vectors.
:ivar capabilities: The capability test vectors.
"""
from __future__ import annotations
from typing import TextIO
from attrs import frozen
from yaml import safe_load, safe_dump
from base64 import b64encode, b64decode
from twisted.python.filepath import FilePath
from .model import Param, Sample, SeedParam
from ..util import CHK, SSK
DATA_PATH: FilePath = FilePath(__file__).sibling("test_vectors.yaml")
# The version of the persisted test vector data this code can interpret.
CURRENT_VERSION: str = "2023-01-16.2"
@frozen
class Case:
"""
Represent one case for which we want/have a test vector.
"""
seed_params: Param
convergence: bytes
seed_data: Sample
fmt: CHK | SSK
segment_size: int
@property
def data(self):
return stretch(self.seed_data.seed, self.seed_data.length)
@property
def params(self):
return self.seed_params.realize(self.fmt.max_shares)
def encode_bytes(b: bytes) -> str:
"""
Base64 encode some bytes to text so they are representable in JSON.
"""
return b64encode(b).decode("ascii")
def decode_bytes(b: str) -> bytes:
"""
Base64 decode some text to bytes.
"""
return b64decode(b.encode("ascii"))
def stretch(seed: bytes, size: int) -> bytes:
"""
Given a simple description of a byte string, return the byte string
itself.
"""
assert isinstance(seed, bytes)
assert isinstance(size, int)
assert size > 0
assert len(seed) > 0
multiples = size // len(seed) + 1
return (seed * multiples)[:size]
def save_capabilities(results: list[tuple[Case, str]], path: FilePath = DATA_PATH) -> None:
"""
Save some test vector cases and their expected values.
This is logically the inverse of ``load_capabilities``.
"""
path.setContent(safe_dump({
"version": CURRENT_VERSION,
"vector": [
{
"convergence": encode_bytes(case.convergence),
"format": {
"kind": case.fmt.kind,
"params": case.fmt.to_json(),
},
"sample": {
"seed": encode_bytes(case.seed_data.seed),
"length": case.seed_data.length,
},
"zfec": {
"segmentSize": case.segment_size,
"required": case.params.required,
"total": case.params.total,
},
"expected": cap,
}
for (case, cap)
in results
],
}).encode("ascii"))
def load_format(serialized: dict) -> CHK | SSK:
"""
Load an encrypted object format from a simple description of it.
:param serialized: A ``dict`` describing either CHK or SSK, possibly with
some parameters.
"""
if serialized["kind"] == "chk":
return CHK.load(serialized["params"])
elif serialized["kind"] == "ssk":
return SSK.load(serialized["params"])
else:
raise ValueError(f"Unrecognized format: {serialized}")
def load_capabilities(f: TextIO) -> dict[Case, str]:
"""
Load some test vector cases and their expected results from the given
file.
This is logically the inverse of ``save_capabilities``.
"""
data = safe_load(f)
if data is None:
return {}
if data["version"] != CURRENT_VERSION:
print(
f"Current version is {CURRENT_VERSION}; "
f"cannot load version {data['version']} data."
)
return {}
return {
Case(
seed_params=SeedParam(case["zfec"]["required"], case["zfec"]["total"]),
segment_size=case["zfec"]["segmentSize"],
convergence=decode_bytes(case["convergence"]),
seed_data=Sample(decode_bytes(case["sample"]["seed"]), case["sample"]["length"]),
fmt=load_format(case["format"]),
): case["expected"]
for case
in data["vector"]
}
try:
with DATA_PATH.open() as f:
capabilities: dict[Case, str] = load_capabilities(f)
except FileNotFoundError:
capabilities = {}

36
misc/windows-enospc/passthrough.py Normal file
View File

@ -0,0 +1,36 @@
"""
Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on
Windows. So, this program passes through data from stdin to stdout, using
Windows APIs instead of Unix-y APIs.
"""
from twisted.internet.stdio import StandardIO
from twisted.internet import reactor
from twisted.internet.protocol import Protocol
from twisted.internet.interfaces import IHalfCloseableProtocol
from twisted.internet.error import ReactorNotRunning
from zope.interface import implementer
@implementer(IHalfCloseableProtocol)
class Passthrough(Protocol):
def readConnectionLost(self):
self.transport.loseConnection()
def writeConnectionLost(self):
try:
reactor.stop()
except ReactorNotRunning:
pass
def dataReceived(self, data):
self.transport.write(data)
def connectionLost(self, reason):
try:
reactor.stop()
except ReactorNotRunning:
pass
std = StandardIO(Passthrough())
reactor.run()

0
newsfragments/3914.minor Normal file
View File

0
newsfragments/3958.minor Normal file
View File

0
newsfragments/3960.minor Normal file
View File

1
newsfragments/3961.other Normal file
View File

@ -0,0 +1 @@
The integration test suite now includes a set of capability test vectors (``integration/vectors/test_vectors.yaml``) which can be used to verify compatibility between Tahoe-LAFS and other implementations.

1
newsfragments/3962.feature Normal file
View File

@ -0,0 +1 @@
Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties.

1
newsfragments/3964.removed Normal file
View File

@ -0,0 +1 @@
Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested.

1
newsfragments/3966.bugfix Normal file
View File

@ -0,0 +1 @@
Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller.

3
pytest.ini Normal file
View File

@ -0,0 +1,3 @@
[pytest]
markers =
slow: marks tests as slow (not run by default; run them with '--runslow')

23
setup.py
View File

@ -139,12 +139,22 @@ install_requires = [
"werkzeug != 2.2.0",
"treq",
"cbor2",
# Need 0.4 to be able to pass in mmap()
"pycddl >= 0.4",
# Ideally we want 0.4+ to be able to pass in mmap(), but it's not strictly
# necessary yet until we fix the workaround to
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963 in
# allmydata.storage.http_server.
"pycddl",
# for pid-file support
"psutil",
"filelock",
# treq needs requests, requests needs charset_normalizer,
# charset_normalizer breaks PyInstaller
# (https://github.com/Ousret/charset_normalizer/issues/253). So work around
# this by using a lower version number. Once upstream issue is fixed, or
# requests drops charset_normalizer, this can go away.
"charset_normalizer < 3",
]
setup_requires = [
@ -224,7 +234,7 @@ def run_command(args, cwd=None):
use_shell = sys.platform == "win32"
try:
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell)
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+
except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+
print("Warning: unable to run %r." % (" ".join(args),))
print(e)
return None
@ -375,8 +385,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
package_dir = {'':'src'},
packages=find_packages('src') + ['allmydata.test.plugins'],
classifiers=trove_classifiers,
# We support Python 3.7 or later. 3.11 is not supported yet.
python_requires=">=3.7, <3.11",
# We support Python 3.8 or later. 3.11 is not supported yet.
python_requires=">=3.8, <3.11",
install_requires=install_requires,
extras_require={
# Duplicate the Twisted pywin32 dependency here. See
@ -389,9 +399,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
],
"test": [
"flake8",
# On Python 3.7, importlib_metadata v5 breaks flake8.
# https://github.com/python/importlib_metadata/issues/407
"importlib_metadata<5; python_version < '3.8'",
# Pin a specific pyflakes so we don't have different folks
# disagreeing on what is or is not a lint issue. We can bump
# this version from time to time, but we will do it

36
src/allmydata/client.py
View File

@ -32,6 +32,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer
from allmydata import storage_client
from allmydata.immutable.upload import Uploader
from allmydata.immutable.offloaded import Helper
from allmydata.mutable.filenode import MutableFileNode
from allmydata.introducer.client import IntroducerClient
from allmydata.util import (
hashutil, base32, pollmixin, log, idlib,
@ -1086,9 +1087,40 @@ class _Client(node.Node, pollmixin.PollMixin):
def create_immutable_dirnode(self, children, convergence=None):
return self.nodemaker.create_immutable_directory(children, convergence)
def create_mutable_file(self, contents=None, version=None):
def create_mutable_file(
self,
contents: bytes | None = None,
version: int | None = None,
*,
unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
) -> MutableFileNode:
"""
Create *and upload* a new mutable object.
:param contents: If given, the initial contents for the new object.
:param version: If given, the mutable file format for the new object
(otherwise a format will be chosen automatically).
:param unique_keypair: **Warning** This value independently determines
the identity of the mutable object to create. There cannot be two
different mutable objects that share a keypair. They will merge
into one object (with undefined contents).
It is common to pass a None value (or not pass a valuye) for this
parameter. In these cases, a new random keypair will be
generated.
If non-None, the given public/private keypair will be used for the
new object. The expected use-case is for implementing compliance
tests.
:return: A Deferred which will fire with a representation of the new
mutable object after it has been uploaded.
"""
return self.nodemaker.create_mutable_file(contents,
version=version)
version=version,
keypair=unique_keypair)
def upload(self, uploadable, reactor=None):
uploader = self.getServiceNamed("uploader")

86
src/allmydata/crypto/rsa.py
View File

@ -9,17 +9,14 @@ features of any objects that `cryptography` documents.
That is, the public and private keys are opaque objects; DO NOT depend
on any of their methods.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing_extensions import TypeAlias
from typing import Callable
from functools import partial
from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.backends import default_backend
@ -30,6 +27,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l
from allmydata.crypto.error import BadSignature
PublicKey: TypeAlias = rsa.RSAPublicKey
PrivateKey: TypeAlias = rsa.RSAPrivateKey
# This is the value that was used by `pycryptopp`, and we must continue to use it for
# both backwards compatibility and interoperability.
@ -46,12 +45,12 @@ RSA_PADDING = padding.PSS(
def create_signing_keypair(key_size):
def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]:
"""
Create a new RSA signing (private) keypair from scratch. Can be used with
`sign_data` function.
:param int key_size: length of key in bits
:param key_size: length of key in bits
:returns: 2-tuple of (private_key, public_key)
"""
@ -63,32 +62,62 @@ def create_signing_keypair(key_size):
return priv_key, priv_key.public_key()
def create_signing_keypair_from_string(private_key_der):
def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]:
"""
Create an RSA signing (private) key from previously serialized
private key bytes.
:param bytes private_key_der: blob as returned from `der_string_from_signing_keypair`
:param private_key_der: blob as returned from `der_string_from_signing_keypair`
:returns: 2-tuple of (private_key, public_key)
"""
priv_key = load_der_private_key(
_load = partial(
load_der_private_key,
private_key_der,
password=None,
backend=default_backend(),
)
if not isinstance(priv_key, rsa.RSAPrivateKey):
def load_with_validation() -> PrivateKey:
k = _load()
assert isinstance(k, PrivateKey)
return k
def load_without_validation() -> PrivateKey:
k = _load(unsafe_skip_rsa_key_validation=True)
assert isinstance(k, PrivateKey)
return k
# Load it once without the potentially expensive OpenSSL validation
# checks. These have superlinear complexity. We *will* run them just
# below - but first we'll apply our own constant-time checks.
load: Callable[[], PrivateKey] = load_without_validation
try:
unsafe_priv_key = load()
except TypeError:
# cryptography<39 does not support this parameter, so just load the
# key with validation...
unsafe_priv_key = load_with_validation()
# But avoid *reloading* it since that will run the expensive
# validation *again*.
load = lambda: unsafe_priv_key
if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey):
raise ValueError(
"Private Key did not decode to an RSA key"
)
if priv_key.key_size != 2048:
if unsafe_priv_key.key_size != 2048:
raise ValueError(
"Private Key must be 2048 bits"
)
return priv_key, priv_key.public_key()
# Now re-load it with OpenSSL's validation applied.
safe_priv_key = load()
return safe_priv_key, safe_priv_key.public_key()
def der_string_from_signing_key(private_key):
def der_string_from_signing_key(private_key: PrivateKey) -> bytes:
"""
Serializes a given RSA private key to a DER string
@ -98,14 +127,14 @@ def der_string_from_signing_key(private_key):
:returns: bytes representing `private_key`
"""
_validate_private_key(private_key)
return private_key.private_bytes(
return private_key.private_bytes( # type: ignore[attr-defined]
encoding=Encoding.DER,
format=PrivateFormat.PKCS8,
encryption_algorithm=NoEncryption(),
)
def der_string_from_verifying_key(public_key):
def der_string_from_verifying_key(public_key: PublicKey) -> bytes:
"""
Serializes a given RSA public key to a DER string.
@ -121,7 +150,7 @@ def der_string_from_verifying_key(public_key):
)
def create_verifying_key_from_string(public_key_der):
def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey:
"""
Create an RSA verifying key from a previously serialized public key
@ -134,15 +163,16 @@ def create_verifying_key_from_string(public_key_der):
public_key_der,
backend=default_backend(),
)
assert isinstance(pub_key, PublicKey)
return pub_key
def sign_data(private_key, data):
def sign_data(private_key: PrivateKey, data: bytes) -> bytes:
"""
:param private_key: the private part of a keypair returned from
`create_signing_keypair_from_string` or `create_signing_keypair`
:param bytes data: the bytes to sign
:param data: the bytes to sign
:returns: bytes which are a signature of the bytes given as `data`.
"""
@ -153,7 +183,7 @@ def sign_data(private_key, data):
hashes.SHA256(),
)
def verify_signature(public_key, alleged_signature, data):
def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None:
"""
:param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key`
@ -173,23 +203,23 @@ def verify_signature(public_key, alleged_signature, data):
raise BadSignature()
def _validate_public_key(public_key):
def _validate_public_key(public_key: PublicKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(public_key, rsa.RSAPublicKey):
raise ValueError(
"public_key must be an RSAPublicKey"
f"public_key must be an RSAPublicKey not {type(public_key)}"
)
def _validate_private_key(private_key):
def _validate_private_key(private_key: PrivateKey) -> None:
"""
Internal helper. Checks that `public_key` is a valid cryptography
object
"""
if not isinstance(private_key, rsa.RSAPrivateKey):
raise ValueError(
"private_key must be an RSAPrivateKey"
f"private_key must be an RSAPrivateKey not {type(private_key)}"
)

42
src/allmydata/mutable/common.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
MODE_CHECK = "MODE_CHECK" # query all peers
MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version
@ -17,6 +10,9 @@ MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial
MODE_READ = "MODE_READ"
MODE_REPAIR = "MODE_REPAIR" # query all peers, get the privkey
from allmydata.crypto import aes, rsa
from allmydata.util import hashutil
class NotWriteableError(Exception):
pass
@ -68,3 +64,33 @@ class CorruptShareError(BadShareError):
class UnknownVersionError(BadShareError):
"""The share we received was of a version we don't recognize."""
def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes:
"""
For SSK, encrypt a private ("signature") key using the writekey.
"""
encryptor = aes.create_encryptor(writekey)
crypttext = aes.encrypt_data(encryptor, privkey)
return crypttext
def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey:
"""
The inverse of ``encrypt_privkey``.
"""
decryptor = aes.create_decryptor(writekey)
privkey = aes.decrypt_data(decryptor, enc_privkey)
return privkey
def derive_mutable_keys(keypair: tuple[rsa.PublicKey, rsa.PrivateKey]) -> tuple[bytes, bytes, bytes]:
"""
Derive the SSK writekey, encrypted writekey, and fingerprint from the
public/private ("verification" / "signature") keypair.
"""
pubkey, privkey = keypair
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
privkey_s = rsa.der_string_from_signing_key(privkey)
writekey = hashutil.ssk_writekey_hash(privkey_s)
encprivkey = encrypt_privkey(writekey, privkey_s)
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
return writekey, encprivkey, fingerprint

42
src/allmydata/mutable/filenode.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import random
@ -16,8 +9,6 @@ from zope.interface import implementer
from twisted.internet import defer, reactor
from foolscap.api import eventually
from allmydata.crypto import aes
from allmydata.crypto import rsa
from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \
NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \
IMutableFileVersion, IWriteable
@ -28,8 +19,14 @@ from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \
from allmydata.monitor import Monitor
from allmydata.mutable.publish import Publish, MutableData,\
TransformingUploadable
from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \
UncoordinatedWriteError
from allmydata.mutable.common import (
MODE_READ,
MODE_WRITE,
MODE_CHECK,
UnrecoverableFileError,
UncoordinatedWriteError,
derive_mutable_keys,
)
from allmydata.mutable.servermap import ServerMap, ServermapUpdater
from allmydata.mutable.retrieve import Retrieve
from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer
@ -139,13 +136,10 @@ class MutableFileNode(object):
Deferred that fires (with the MutableFileNode instance you should
use) when it completes.
"""
(pubkey, privkey) = keypair
self._pubkey, self._privkey = pubkey, privkey
pubkey_s = rsa.der_string_from_verifying_key(self._pubkey)
privkey_s = rsa.der_string_from_signing_key(self._privkey)
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
self._pubkey, self._privkey = keypair
self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys(
keypair,
)
if version == MDMF_VERSION:
self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
self._protocol_version = version
@ -171,16 +165,6 @@ class MutableFileNode(object):
(contents, type(contents))
return contents(self)
def _encrypt_privkey(self, writekey, privkey):
encryptor = aes.create_encryptor(writekey)
crypttext = aes.encrypt_data(encryptor, privkey)
return crypttext
def _decrypt_privkey(self, enc_privkey):
decryptor = aes.create_decryptor(self._writekey)
privkey = aes.decrypt_data(decryptor, enc_privkey)
return privkey
def _populate_pubkey(self, pubkey):
self._pubkey = pubkey
def _populate_required_shares(self, required_shares):

17
src/allmydata/mutable/retrieve.py
View File

@ -1,15 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Don't import bytes and str, to prevent API leakage
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401
from __future__ import annotations
import time
@ -32,7 +24,7 @@ from allmydata import hashtree, codec
from allmydata.storage.server import si_b2a
from allmydata.mutable.common import CorruptShareError, BadShareError, \
UncoordinatedWriteError
UncoordinatedWriteError, decrypt_privkey
from allmydata.mutable.layout import MDMFSlotReadProxy
@implementer(IRetrieveStatus)
@ -931,9 +923,10 @@ class Retrieve(object):
def _try_to_validate_privkey(self, enc_privkey, reader, server):
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
node_writekey = self._node.get_writekey()
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
if alleged_writekey != node_writekey:
self.log("invalid privkey from %s shnum %d" %
(reader, reader.shnum),
level=log.WEIRD, umid="YIw4tA")

27
src/allmydata/mutable/servermap.py
View File

@ -1,16 +1,8 @@
"""
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
# Doesn't import str to prevent API leakage on Python 2
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
from past.builtins import unicode
from six import ensure_str
import sys, time, copy
@ -29,7 +21,7 @@ from allmydata.storage.server import si_b2a
from allmydata.interfaces import IServermapUpdaterStatus
from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \
MODE_READ, MODE_REPAIR, CorruptShareError
MODE_READ, MODE_REPAIR, CorruptShareError, decrypt_privkey
from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy
@implementer(IServermapUpdaterStatus)
@ -203,8 +195,8 @@ class ServerMap(object):
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = verinfo
print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" %
(unicode(server.get_name(), "utf-8"), shnum,
seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N,
(str(server.get_name(), "utf-8"), shnum,
seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), k, N,
datalength), file=out)
if self._problems:
print("%d PROBLEMS" % len(self._problems), file=out)
@ -276,7 +268,7 @@ class ServerMap(object):
"""Take a versionid, return a string that describes it."""
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
offsets_tuple) = verinfo
return "seq%d-%s" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"))
return "seq%d-%s" % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"))
def summarize_versions(self):
"""Return a string describing which versions we know about."""
@ -824,7 +816,7 @@ class ServermapUpdater(object):
def notify_server_corruption(self, server, shnum, reason):
if isinstance(reason, unicode):
if isinstance(reason, str):
reason = reason.encode("utf-8")
ss = server.get_storage_server()
ss.advise_corrupt_share(
@ -879,7 +871,7 @@ class ServermapUpdater(object):
# ok, it's a valid verinfo. Add it to the list of validated
# versions.
self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d"
% (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"),
% (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"),
ensure_str(server.get_name()), shnum,
k, n, segsize, datalen),
parent=lp)
@ -951,9 +943,10 @@ class ServermapUpdater(object):
writekey stored in my node. If it is valid, then I set the
privkey and encprivkey properties of the node.
"""
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
node_writekey = self._node.get_writekey()
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
if alleged_writekey != self._node.get_writekey():
if alleged_writekey != node_writekey:
self.log("invalid privkey from %r shnum %d" %
(server.get_name(), shnum),
parent=lp, level=log.WEIRD, umid="aJVccw")

19
src/allmydata/nodemaker.py
View File

@ -1,17 +1,12 @@
"""
Ported to Python 3.
Create file nodes of various types.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import weakref
from zope.interface import implementer
from twisted.internet.defer import succeed
from allmydata.util.assertutil import precondition
from allmydata.interfaces import INodeMaker
from allmydata.immutable.literal import LiteralFileNode
@ -22,6 +17,7 @@ from allmydata.mutable.publish import MutableData
from allmydata.dirnode import DirectoryNode, pack_children
from allmydata.unknown import UnknownNode
from allmydata.blacklist import ProhibitedNode
from allmydata.crypto.rsa import PublicKey, PrivateKey
from allmydata import uri
@ -126,12 +122,15 @@ class NodeMaker(object):
return self._create_dirnode(filenode)
return None
def create_mutable_file(self, contents=None, version=None):
def create_mutable_file(self, contents=None, version=None, keypair: tuple[PublicKey, PrivateKey] | None = None):
if version is None:
version = self.mutable_file_default
n = MutableFileNode(self.storage_broker, self.secret_holder,
self.default_encoding_parameters, self.history)
d = self.key_generator.generate()
if keypair is None:
d = self.key_generator.generate()
else:
d = succeed(keypair)
d.addCallback(n.create_with_keys, contents, version=version)
d.addCallback(lambda res: n)
return d

16
src/allmydata/scripts/cli.py
View File

@ -180,10 +180,22 @@ class GetOptions(FileStoreOptions):
class PutOptions(FileStoreOptions):
optFlags = [
("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
]
]
optParameters = [
("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
]
("private-key-path", None, None,
"***Warning*** "
"It is possible to use this option to spoil the normal security properties of mutable objects. "
"It is also possible to corrupt or destroy data with this option. "
"Most users will not need this option and can ignore it. "
"For mutables only, "
"this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. "
"The private key must be handled at least as strictly as the resulting capability string. "
"A single private key must not be used for more than one mutable."
),
]
def parseArgs(self, arg1=None, arg2=None):
# see Examples below

42
src/allmydata/scripts/tahoe_put.py
View File

@ -1,23 +1,32 @@
"""
Ported to Python 3.
Implement the ``tahoe put`` command.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from io import BytesIO
from urllib.parse import quote as url_quote
from base64 import urlsafe_b64encode
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from twisted.python.filepath import FilePath
from allmydata.crypto.rsa import PrivateKey, der_string_from_signing_key
from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
UnknownAliasError
from allmydata.util.encodingutil import quote_output
def load_private_key(path: str) -> str:
"""
Load a private key from a file and return it in a format appropriate
to include in the HTTP request.
"""
privkey = load_pem_private_key(FilePath(path).getContent(), password=None)
assert isinstance(privkey, PrivateKey)
derbytes = der_string_from_signing_key(privkey)
return urlsafe_b64encode(derbytes).decode("ascii")
def put(options):
"""
@param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
@ -29,6 +38,10 @@ def put(options):
from_file = options.from_file
to_file = options.to_file
mutable = options['mutable']
if options["private-key-path"] is None:
private_key = None
else:
private_key = load_private_key(options["private-key-path"])
format = options['format']
if options['quiet']:
verbosity = 0
@ -79,6 +92,12 @@ def put(options):
queryargs = []
if mutable:
queryargs.append("mutable=true")
if private_key is not None:
queryargs.append(f"private-key={private_key}")
else:
if private_key is not None:
raise Exception("Can only supply a private key for mutables.")
if format:
queryargs.append("format=%s" % format)
if queryargs:
@ -92,10 +111,7 @@ def put(options):
if verbosity > 0:
print("waiting for file data on stdin..", file=stderr)
# We're uploading arbitrary files, so this had better be bytes:
if PY2:
stdinb = stdin
else:
stdinb = stdin.buffer
stdinb = stdin.buffer
data = stdinb.read()
infileobj = BytesIO(data)

17
src/allmydata/storage/http_server.py
View File

@ -11,6 +11,7 @@ import binascii
from tempfile import TemporaryFile
from os import SEEK_END, SEEK_SET
import mmap
from importlib.metadata import version as get_package_version, PackageNotFoundError
from cryptography.x509 import Certificate as CryptoCertificate
from zope.interface import implementer
@ -62,6 +63,20 @@ from ..util.deferredutil import async_to_deferred
from allmydata.interfaces import BadWriteEnablerError
# Until we figure out Nix (https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3963),
# need to support old pycddl which can only take bytes:
from distutils.version import LooseVersion
try:
PYCDDL_BYTES_ONLY = LooseVersion(get_package_version("pycddl")) < LooseVersion(
"0.4"
)
except PackageNotFoundError:
# This can happen when building PyInstaller distribution. We'll just assume
# you installed a modern pycddl, cause why wouldn't you?
PYCDDL_BYTES_ONLY = False
class ClientSecretsException(Exception):
"""The client did not send the appropriate secrets."""
@ -561,7 +576,7 @@ class HTTPServer(object):
fd = request.content.fileno()
except (ValueError, OSError):
fd = -1
if fd > 0:
if fd >= 0 and not PYCDDL_BYTES_ONLY:
# It's a file, so we can use mmap() to save memory.
message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ)
else:

210
src/allmydata/test/_win_subprocess.py
View File

@ -1,210 +0,0 @@
"""
This module is only necessary on Python 2. Once Python 2 code is dropped, it
can be deleted.
"""
from future.utils import PY3
if PY3:
raise RuntimeError("Just use subprocess.Popen")
# This is necessary to pacify flake8 on Python 3, while we're still supporting
# Python 2.
from past.builtins import unicode
# -*- coding: utf-8 -*-
## Copyright (C) 2021 Valentin Lab
##
## Redistribution and use in source and binary forms, with or without
## modification, are permitted provided that the following conditions
## are met:
##
## 1. Redistributions of source code must retain the above copyright
## notice, this list of conditions and the following disclaimer.
##
## 2. Redistributions in binary form must reproduce the above
## copyright notice, this list of conditions and the following
## disclaimer in the documentation and/or other materials provided
## with the distribution.
##
## 3. Neither the name of the copyright holder nor the names of its
## contributors may be used to endorse or promote products derived
## from this software without specific prior written permission.
##
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
## OF THE POSSIBILITY OF SUCH DAMAGE.
##
## issue: https://bugs.python.org/issue19264
# See allmydata/windows/fixups.py
import sys
assert sys.platform == "win32"
import os
import ctypes
import subprocess
import _subprocess
from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \
Structure, sizeof, c_wchar, WinError
from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \
HANDLE
##
## Types
##
CREATE_UNICODE_ENVIRONMENT = 0x00000400
LPCTSTR = c_char_p
LPTSTR = c_wchar_p
LPSECURITY_ATTRIBUTES = c_void_p
LPBYTE = ctypes.POINTER(BYTE)
class STARTUPINFOW(Structure):
_fields_ = [
("cb", DWORD), ("lpReserved", LPWSTR),
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
("dwX", DWORD), ("dwY", DWORD),
("dwXSize", DWORD), ("dwYSize", DWORD),
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
("wShowWindow", WORD), ("cbReserved2", WORD),
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
("hStdOutput", HANDLE), ("hStdError", HANDLE),
]
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
class PROCESS_INFORMATION(Structure):
_fields_ = [
("hProcess", HANDLE), ("hThread", HANDLE),
("dwProcessId", DWORD), ("dwThreadId", DWORD),
]
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
class DUMMY_HANDLE(ctypes.c_void_p):
def __init__(self, *a, **kw):
super(DUMMY_HANDLE, self).__init__(*a, **kw)
self.closed = False
def Close(self):
if not self.closed:
windll.kernel32.CloseHandle(self)
self.closed = True
def __int__(self):
return self.value
CreateProcessW = windll.kernel32.CreateProcessW
CreateProcessW.argtypes = [
LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES,
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR,
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
]
CreateProcessW.restype = BOOL
##
## Patched functions/classes
##
def CreateProcess(executable, args, _p_attr, _t_attr,
inherit_handles, creation_flags, env, cwd,
startup_info):
"""Create a process supporting unicode executable and args for win32
Python implementation of CreateProcess using CreateProcessW for Win32
"""
si = STARTUPINFOW(
dwFlags=startup_info.dwFlags,
wShowWindow=startup_info.wShowWindow,
cb=sizeof(STARTUPINFOW),
## XXXvlab: not sure of the casting here to ints.
hStdInput=int(startup_info.hStdInput),
hStdOutput=int(startup_info.hStdOutput),
hStdError=int(startup_info.hStdError),
)
wenv = None
if env is not None:
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
env = (unicode("").join([
unicode("%s=%s\0") % (k, v)
for k, v in env.items()])) + unicode("\0")
wenv = (c_wchar * len(env))()
wenv.value = env
pi = PROCESS_INFORMATION()
creation_flags |= CREATE_UNICODE_ENVIRONMENT
if CreateProcessW(executable, args, None, None,
inherit_handles, creation_flags,
wenv, cwd, byref(si), byref(pi)):
return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
pi.dwProcessId, pi.dwThreadId)
raise WinError()
class Popen(subprocess.Popen):
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell, to_close,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
There are only 2 little changes concerning the construction of
the the final string in shell mode: we preempt the creation of
the command string when shell is True, because original function
will try to encode unicode args which we want to avoid to be able to
sending it as-is to ``CreateProcess``.
"""
if not isinstance(args, subprocess.types.StringTypes):
args = subprocess.list2cmdline(args)
if startupinfo is None:
startupinfo = subprocess.STARTUPINFO()
if shell:
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
startupinfo.wShowWindow = _subprocess.SW_HIDE
comspec = os.environ.get("COMSPEC", unicode("cmd.exe"))
args = unicode('{} /c "{}"').format(comspec, args)
if (_subprocess.GetVersion() >= 0x80000000 or
os.path.basename(comspec).lower() == "command.com"):
w9xpopen = self._find_w9xpopen()
args = unicode('"%s" %s') % (w9xpopen, args)
creationflags |= _subprocess.CREATE_NEW_CONSOLE
cp = _subprocess.CreateProcess
_subprocess.CreateProcess = CreateProcess
try:
super(Popen, self)._execute_child(
args, executable,
preexec_fn, close_fds, cwd, env, universal_newlines,
startupinfo, creationflags, False, to_close, p2cread,
p2cwrite, c2pread, c2pwrite, errread, errwrite,
)
finally:
_subprocess.CreateProcess = cp

79
src/allmydata/test/cli/test_put.py
View File

@ -1,19 +1,18 @@
"""
Ported to Python 3.
Tests for the ``tahoe put`` CLI tool.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing import Callable, Awaitable, TypeVar, Any
import os.path
from twisted.trial import unittest
from twisted.python import usage
from twisted.python.filepath import FilePath
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from allmydata.crypto.rsa import PrivateKey
from allmydata.uri import from_string
from allmydata.util import fileutil
from allmydata.scripts.common import get_aliases
from allmydata.scripts import cli
@ -22,6 +21,9 @@ from ..common_util import skip_if_cannot_represent_filename
from allmydata.util.encodingutil import get_io_encoding
from allmydata.util.fileutil import abspath_expanduser_unicode
from .common import CLITestMixin
from allmydata.mutable.common import derive_mutable_keys
T = TypeVar("T")
class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
@ -215,6 +217,65 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
return d
async def test_unlinked_mutable_specified_private_key(self) -> None:
"""
A new unlinked mutable can be created using a specified private
key.
"""
self.basedir = "cli/Put/unlinked-mutable-with-key"
await self._test_mutable_specified_key(
lambda do_cli, pempath, datapath: do_cli(
"put", "--mutable", "--private-key-path", pempath.path,
stdin=datapath.getContent(),
),
)
async def test_linked_mutable_specified_private_key(self) -> None:
"""
A new linked mutable can be created using a specified private key.
"""
self.basedir = "cli/Put/linked-mutable-with-key"
await self._test_mutable_specified_key(
lambda do_cli, pempath, datapath: do_cli(
"put", "--mutable", "--private-key-path", pempath.path, datapath.path,
),
)
async def _test_mutable_specified_key(
self,
run: Callable[[Any, FilePath, FilePath], Awaitable[tuple[int, bytes, bytes]]],
) -> None:
"""
A helper for testing mutable creation.
:param run: A function to do the creation. It is called with
``self.do_cli`` and the path to a private key PEM file and a data
file. It returns whatever ``do_cli`` returns.
"""
self.set_up_grid(oneshare=True)
pempath = FilePath(__file__).parent().sibling("data").child("openssl-rsa-2048.txt")
datapath = FilePath(self.basedir).child("data")
datapath.setContent(b"Hello world" * 1024)
(rc, out, err) = await run(self.do_cli, pempath, datapath)
self.assertEqual(rc, 0, (out, err))
cap = from_string(out.strip())
# The capability is derived from the key we specified.
privkey = load_pem_private_key(pempath.getContent(), password=None)
assert isinstance(privkey, PrivateKey)
pubkey = privkey.public_key()
writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
self.assertEqual(
(writekey, fingerprint),
(cap.writekey, cap.fingerprint),
)
# Also the capability we were given actually refers to the data we
# uploaded.
(rc, out, err) = await self.do_cli("get", out.strip())
self.assertEqual(rc, 0, (out, err))
self.assertEqual(out, datapath.getContent().decode("ascii"))
def test_mutable(self):
# echo DATA1 | tahoe put --mutable - uploaded.txt
# echo DATA2 | tahoe put - uploaded.txt # should modify-in-place

78
src/allmydata/test/common.py
View File

@ -1,14 +1,8 @@
"""
Ported to Python 3.
Functionality related to a lot of the test suite.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2, native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import chr as byteschr
__all__ = [
@ -111,25 +105,15 @@ from allmydata.scripts.common import (
from ..crypto import (
ed25519,
rsa,
)
from .eliotutil import (
EliotLoggedRunTest,
)
from .common_util import ShouldFailMixin # noqa: F401
if sys.platform == "win32" and PY2:
# Python 2.7 doesn't have good options for launching a process with
# non-ASCII in its command line. So use this alternative that does a
# better job. However, only use it on Windows because it doesn't work
# anywhere else.
from ._win_subprocess import (
Popen,
)
else:
from subprocess import (
Popen,
)
from subprocess import (
Popen,
PIPE,
)
@ -298,7 +282,7 @@ class UseNode(object):
plugin_config = attr.ib()
storage_plugin = attr.ib()
basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str),
introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
converter=six.ensure_str)
node_config = attr.ib(default=attr.Factory(dict))
@ -639,15 +623,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
MUTABLE_SIZELIMIT = 10000
def __init__(self, storage_broker, secret_holder,
default_encoding_parameters, history, all_contents):
_public_key: rsa.PublicKey | None
_private_key: rsa.PrivateKey | None
def __init__(self,
storage_broker,
secret_holder,
default_encoding_parameters,
history,
all_contents,
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None
):
self.all_contents = all_contents
self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
self.init_from_cap(make_mutable_file_cap())
self.file_types: dict[bytes, int] = {} # storage index => MDMF_VERSION or SDMF_VERSION
self.init_from_cap(make_mutable_file_cap(keypair))
self._k = default_encoding_parameters['k']
self._segsize = default_encoding_parameters['max_segment_size']
def create(self, contents, key_generator=None, keysize=None,
version=SDMF_VERSION):
if keypair is None:
self._public_key = self._private_key = None
else:
self._public_key, self._private_key = keypair
def create(self, contents, version=SDMF_VERSION):
if version == MDMF_VERSION and \
isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
uri.WriteableSSKFileURI)):
@ -843,9 +840,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
return defer.succeed(consumer)
def make_mutable_file_cap():
return uri.WriteableSSKFileURI(writekey=os.urandom(16),
fingerprint=os.urandom(32))
def make_mutable_file_cap(
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
) -> uri.WriteableSSKFileURI:
"""
Create a local representation of a mutable object.
:param keypair: If None, a random keypair will be generated for the new
object. Otherwise, this is the keypair for that object.
"""
if keypair is None:
writekey = os.urandom(16)
fingerprint = os.urandom(32)
else:
pubkey, privkey = keypair
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
privkey_s = rsa.der_string_from_signing_key(privkey)
writekey = hashutil.ssk_writekey_hash(privkey_s)
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
return uri.WriteableSSKFileURI(
writekey=writekey, fingerprint=fingerprint,
)
def make_mdmf_mutable_file_cap():
return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
@ -875,7 +891,7 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
encoding_params['max_segment_size'] = 128*1024
filenode = FakeMutableFileNode(None, None, encoding_params, None,
all_contents)
all_contents, None)
filenode.init_from_cap(cap)
if mdmf:
filenode.create(MutableData(contents), version=MDMF_VERSION)

28
src/allmydata/test/data/openssl-rsa-2048.txt Normal file
View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDF1MeXulDWFO05
YXCh8aqNc1dS1ddJRzsti4BOWuDOepUc0oCaSIcC5aR7XJ+vhX7a02mTIwvLcuEH
8sxx0BJU4jCDpRI6aAqaKJxwZx1e6AcVFJDl7vzymhvWhqHuKh0jTvwM2zONWTwV
V8m2PbDdxu0Prwdx+Mt2sDT6xHEhJj5fI/GUDUEdkhLJF6DQSulFRqqd0qP7qcI9
fSHZbM7MywfzqFUe8J1+tk4fBh2v7gNzN1INpzh2mDtLPAtxr4ZPtEb/0D0U4PsP
CniOHP0U8sF3VY0+K5qoCQr92cLRJvT/vLpQGVNUTFdFrtbqDoFxUCyEH4FUqRDX
2mVrPo2xAgMBAAECggEAA0Ev1y5/1NTPbgytBeIIH3d+v9hwKDbHecVoMwnOVeFJ
BZpONrOToovhAc1NXH2wj4SvwYWfpJ1HR9piDAuLeKlnuUu4ffzfE0gQok4E+v4r
2yg9ZcYBs/NOetAYVwbq960tiv/adFRr71E0WqbfS3fBx8q2L3Ujkkhd98PudUhQ
izbrTvkT7q00OPCWGwgWepMlLEowUWwZehGI0MlbONg7SbRraZZmG586Iy0tpC3e
AM7wC1/ORzFqcRgTIxXizQ5RHL7S0OQPLhbEJbuwPonNjze3p0EP4wNBELZTaVOd
xeA22Py4Bh/d1q3aEgbwR7tLyA8YfEzshTaY6oV8AQKBgQD0uFo8pyWk0AWXfjzn
jV4yYyPWy8pJA6YfAJAST8m7B/JeYgGlfHxTlNZiB40DsJq08tOZv3HAubgMpFIa
reuDxPqo6/Quwdy4Syu+AFhY48KIuwuoegG/L+5qcQLE69r1w71ZV6wUvLmXYX2I
Y6nYz+OdpD1JrMIr6Js60XURsQKBgQDO8yWl7ufIDKMbQpbs0PgUQsH4FtzGcP4J
j/7/8GfhKYt6rPsrojPHUbAi1+25xBVOuhm0Zx2ku2t+xPIMJoS+15EcER1Z2iHZ
Zci9UGpJpUxGcUhG7ETF1HZv0xKHcEOl9eIIOcAP9Vd9DqnGk85gy6ti6MHe/5Tn
IMD36OQ8AQKBgQDwqE7NMM67KnslRNaeG47T3F0FQbm3XehCuqnz6BUJYcI+gQD/
fdFB3K+LDcPmKgmqAtaGbxdtoPXXMM0xQXHHTrH15rxmMu1dK0dj/TDkkW7gSZko
YHtRSdCbSnGfuBXG9GxD7QzkA8g7j3sE4oXIGoDLqRVAW61DwubMy+jlsQKBgGNB
+Zepi1/Gt+BWQt8YpzPIhRIBnShMf3uEphCJdLlo3K4dE2btKBp8UpeTq0CDDJky
5ytAndYp0jf+K/2p59dEuyOUDdjPp5aGnA446JGkB35tzPW/Uoj0C049FVEChl+u
HBhH4peE285uXv2QXNbOOMh6zKmxOfDVI9iDyhwBAoGBAIXq2Ar0zDXXaL3ncEKo
pXt9BZ8OpJo2pvB1t2VPePOwEQ0wdT+H62fKNY47NiF9+LyS541/ps5Qhv6AmiKJ
Z7I0Vb6+sxQljYH/LNW+wc2T/pIAi/7sNcmnlBtZfoVwt99bk2CyoRALPLWHYCkh
c7Tty2bZzDZy6aCX+FGRt5N/
-----END PRIVATE KEY-----

11
src/allmydata/test/mutable/test_filenode.py
View File

@ -30,6 +30,7 @@ from allmydata.mutable.publish import MutableData
from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \
StoppingConsumer, ImmediatelyStoppingConsumer
from .. import common_util as testutil
from ...crypto.rsa import create_signing_keypair
from .util import (
FakeStorage,
make_nodemaker_with_peers,
@ -65,6 +66,16 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin):
d.addCallback(_created)
return d
async def test_create_with_keypair(self):
"""
An SDMF can be created using a given keypair.
"""
(priv, pub) = create_signing_keypair(2048)
node = await self.nodemaker.create_mutable_file(keypair=(pub, priv))
self.assertThat(
(node.get_privkey(), node.get_pubkey()),
Equals((priv, pub)),
)
def test_create_mdmf(self):
d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)

3
src/allmydata/test/test_dirnode.py
View File

@ -1619,7 +1619,8 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation
return defer.succeed(None)
class FakeNodeMaker(NodeMaker):
def create_mutable_file(self, contents=b"", keysize=None, version=None):
def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None):
assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs"
return defer.succeed(FakeMutableFile(contents))
class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573

2
src/allmydata/test/test_system.py
View File

@ -521,9 +521,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def test_mutable_sdmf(self):
"""SDMF mutables can be uploaded, downloaded, and many other things."""
return self._test_mutable(SDMF_VERSION)
def test_mutable_mdmf(self):
"""MDMF mutables can be uploaded, downloaded, and many other things."""
return self._test_mutable(MDMF_VERSION)
def _test_mutable(self, mutable_version):

65
src/allmydata/test/web/test_web.py
View File

@ -1,19 +1,14 @@
"""
Ported to Python 3.
Tests for a bunch of web-related APIs.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_binary
import os.path, re, time
import treq
from urllib.parse import quote as urlquote, unquote as urlunquote
from base64 import urlsafe_b64encode
from bs4 import BeautifulSoup
@ -38,6 +33,7 @@ from allmydata.util import fileutil, base32, hashutil, jsonbytes as json
from allmydata.util.consumer import download_to_data
from allmydata.util.encodingutil import to_bytes
from ...util.connection_status import ConnectionStatus
from ...crypto.rsa import PublicKey, PrivateKey, create_signing_keypair, der_string_from_signing_key
from ..common import (
EMPTY_CLIENT_CONFIG,
FakeCHKFileNode,
@ -65,6 +61,7 @@ from allmydata.interfaces import (
MustBeReadonlyError,
)
from allmydata.mutable import servermap, publish, retrieve
from allmydata.mutable.common import derive_mutable_keys
from .. import common_util as testutil
from ..common_util import TimezoneMixin
from ..common_web import (
@ -93,6 +90,7 @@ class FakeNodeMaker(NodeMaker):
'happy': 7,
'max_segment_size':128*1024 # 1024=KiB
}
all_contents: dict[bytes, object]
def _create_lit(self, cap):
return FakeCHKFileNode(cap, self.all_contents)
def _create_immutable(self, cap):
@ -100,11 +98,19 @@ class FakeNodeMaker(NodeMaker):
def _create_mutable(self, cap):
return FakeMutableFileNode(None, None,
self.encoding_params, None,
self.all_contents).init_from_cap(cap)
def create_mutable_file(self, contents=b"", keysize=None,
version=SDMF_VERSION):
self.all_contents, None).init_from_cap(cap)
def create_mutable_file(self,
contents=None,
version=None,
keypair: tuple[PublicKey, PrivateKey] | None=None,
):
if contents is None:
contents = b""
if version is None:
version = SDMF_VERSION
n = FakeMutableFileNode(None, None, self.encoding_params, None,
self.all_contents)
self.all_contents, keypair)
return n.create(contents, version=version)
class FakeUploader(service.Service):
@ -2868,6 +2874,41 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
"Unknown format: foo",
method="post", data=body, headers=headers)
async def test_POST_upload_keypair(self) -> None:
"""
A *POST* creating a new mutable object may include a *private-key*
query argument giving a urlsafe-base64-encoded RSA private key to use
as the "signature key". The given signature key is used, rather than
a new one being generated.
"""
format = "sdmf"
priv, pub = create_signing_keypair(2048)
encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii")
filename = "predetermined-sdmf"
expected_content = self.NEWFILE_CONTENTS * 100
actual_cap = uri.from_string(await self.POST(
self.public_url +
f"/foo?t=upload&format={format}&private-key={encoded_privkey}",
file=(filename, expected_content),
))
# Ideally we would inspect the private ("signature") and public
# ("verification") keys but they are not made easily accessible here
# (ostensibly because we have a FakeMutableFileNode instead of a real
# one).
#
# So, instead, re-compute the writekey and fingerprint and compare
# those against the capability string.
expected_writekey, _, expected_fingerprint = derive_mutable_keys((pub, priv))
self.assertEqual(
(expected_writekey, expected_fingerprint),
(actual_cap.writekey, actual_cap.fingerprint),
)
# And the capability we got can be used to download the data we
# uploaded.
downloaded_content = await self.GET(f"/uri/{actual_cap.to_string().decode('ascii')}")
self.assertEqual(expected_content, downloaded_content)
def test_POST_upload_format(self):
def _check_upload(ign, format, uri_prefix, fn=None):
filename = format + ".txt"

10
src/allmydata/test/web/test_webish.py
View File

@ -202,6 +202,16 @@ class TahoeLAFSSiteTests(SyncTestCase):
),
)
def test_private_key_censoring(self):
"""
The log event for a request including a **private-key** query
argument has the private key value censored.
"""
self._test_censoring(
b"/uri?uri=URI:CHK:aaa:bbb&private-key=AAAAaaaabbbb==",
b"/uri?uri=[CENSORED]&private-key=[CENSORED]",
)
def test_uri_censoring(self):
"""
The log event for a request for **/uri/<CAP>** has the capability value

55
src/allmydata/web/common.py
View File

@ -1,26 +1,17 @@
"""
Ported to Python 3.
"""
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it
from __future__ import annotations
from six import ensure_str
try:
from typing import Optional, Union, Tuple, Any
except ImportError:
pass
from typing import Optional, Union, TypeVar, overload
from typing_extensions import Literal
import time
import json
from functools import wraps
from base64 import urlsafe_b64decode
from hyperlink import (
DecodedURL,
@ -94,7 +85,7 @@ from allmydata.util.encodingutil import (
to_bytes,
)
from allmydata.util import abbreviate
from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string
class WebError(Exception):
def __init__(self, text, code=http.BAD_REQUEST):
@ -713,8 +704,15 @@ def url_for_string(req, url_string):
)
return url
T = TypeVar("T")
def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
@overload
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> T | bytes: ...
@overload
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ...
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]:
"""Extract an argument from either the query args (req.args) or the form
body fields (req.fields). If multiple=False, this returns a single value
(or the default, which defaults to None), and the query args take
@ -726,13 +724,14 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni
:return: Either bytes or tuple of bytes.
"""
if isinstance(argname, str):
argname = argname.encode("utf-8")
if isinstance(default, str):
default = default.encode("utf-8")
argname_bytes = argname.encode("utf-8")
else:
argname_bytes = argname
results = []
if argname in req.args:
results.extend(req.args[argname])
argname_unicode = str(argname, "utf-8")
if argname_bytes in req.args:
results.extend(req.args[argname_bytes])
argname_unicode = str(argname_bytes, "utf-8")
if req.fields and argname_unicode in req.fields:
value = req.fields[argname_unicode].value
if isinstance(value, str):
@ -742,6 +741,9 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni
return tuple(results)
if results:
return results[0]
if isinstance(default, str):
return default.encode("utf-8")
return default
@ -833,3 +835,14 @@ def abbreviate_time(data):
if s >= 0.001:
return u"%.1fms" % (1000*s)
return u"%.0fus" % (1000000*s)
def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None:
"""
Load a keypair from a urlsafe-base64-encoded RSA private key in the
**private-key** argument of the given request, if there is one.
"""
privkey_der = get_arg(request, "private-key", default=None, multiple=False)
if privkey_der is None:
return None
privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der))
return pubkey, privkey

33
src/allmydata/web/filenode.py
View File

@ -1,23 +1,12 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
# Use native unicode() as str() to prevent leaking futurebytes in ways that
# break string formattin.
from past.builtins import unicode as str
from past.builtins import long
from __future__ import annotations
from twisted.web import http, static
from twisted.internet import defer
from twisted.web.resource import (
Resource, # note: Resource is an old-style class
Resource,
ErrorPage,
)
@ -34,6 +23,7 @@ from allmydata.blacklist import (
)
from allmydata.web.common import (
get_keypair,
boolean_of_arg,
exception_to_child,
get_arg,
@ -56,7 +46,6 @@ from allmydata.web.check_results import (
from allmydata.web.info import MoreInfo
from allmydata.util import jsonbytes as json
class ReplaceMeMixin(object):
def replace_me_with_a_child(self, req, client, replace):
# a new file is being uploaded in our place.
@ -64,7 +53,8 @@ class ReplaceMeMixin(object):
mutable_type = get_mutable_type(file_format)
if mutable_type is not None:
data = MutableFileHandle(req.content)
d = client.create_mutable_file(data, version=mutable_type)
keypair = get_keypair(req)
d = client.create_mutable_file(data, version=mutable_type, unique_keypair=keypair)
def _uploaded(newnode):
d2 = self.parentnode.set_node(self.name, newnode,
overwrite=replace)
@ -106,7 +96,8 @@ class ReplaceMeMixin(object):
if file_format in ("SDMF", "MDMF"):
mutable_type = get_mutable_type(file_format)
uploadable = MutableFileHandle(contents.file)
d = client.create_mutable_file(uploadable, version=mutable_type)
keypair = get_keypair(req)
d = client.create_mutable_file(uploadable, version=mutable_type, unique_keypair=keypair)
def _uploaded(newnode):
d2 = self.parentnode.set_node(self.name, newnode,
overwrite=replace)
@ -395,7 +386,7 @@ class FileDownloader(Resource, object):
# list of (first,last) inclusive range tuples.
filesize = self.filenode.get_size()
assert isinstance(filesize, (int,long)), filesize
assert isinstance(filesize, int), filesize
try:
# byte-ranges-specifier
@ -408,19 +399,19 @@ class FileDownloader(Resource, object):
if first == '':
# suffix-byte-range-spec
first = filesize - long(last)
first = filesize - int(last)
last = filesize - 1
else:
# byte-range-spec
# first-byte-pos
first = long(first)
first = int(first)
# last-byte-pos
if last == '':
last = filesize - 1
else:
last = long(last)
last = int(last)
if last < first:
raise ValueError
@ -456,7 +447,7 @@ class FileDownloader(Resource, object):
b'attachment; filename="%s"' % self.filename)
filesize = self.filenode.get_size()
assert isinstance(filesize, (int,long)), filesize
assert isinstance(filesize, int), filesize
first, size = 0, None
contentsize = filesize
req.setHeader("accept-ranges", "bytes")

13
src/allmydata/web/unlinked.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from urllib.parse import quote as urlquote
@ -25,6 +18,7 @@ from twisted.web.template import (
from allmydata.immutable.upload import FileHandle
from allmydata.mutable.publish import MutableFileHandle
from allmydata.web.common import (
get_keypair,
get_arg,
boolean_of_arg,
convert_children_json,
@ -48,7 +42,8 @@ def PUTUnlinkedSSK(req, client, version):
# SDMF: files are small, and we can only upload data
req.content.seek(0)
data = MutableFileHandle(req.content)
d = client.create_mutable_file(data, version=version)
keypair = get_keypair(req)
d = client.create_mutable_file(data, version=version, unique_keypair=keypair)
d.addCallback(lambda n: n.get_uri())
return d

98
src/allmydata/webish.py
View File

@ -1,18 +1,12 @@
"""
Ported to Python 3.
General web server-related utilities.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from six import ensure_str
import re, time, tempfile
from urllib.parse import parse_qsl, urlencode
from cgi import (
FieldStorage,
@ -45,40 +39,37 @@ from .web.storage_plugins import (
)
if PY2:
FileUploadFieldStorage = FieldStorage
else:
class FileUploadFieldStorage(FieldStorage):
"""
Do terrible things to ensure files are still bytes.
class FileUploadFieldStorage(FieldStorage):
"""
Do terrible things to ensure files are still bytes.
On Python 2, uploaded files were always bytes. On Python 3, there's a
heuristic: if the filename is set on a field, it's assumed to be a file
upload and therefore bytes. If no filename is set, it's Unicode.
On Python 2, uploaded files were always bytes. On Python 3, there's a
heuristic: if the filename is set on a field, it's assumed to be a file
upload and therefore bytes. If no filename is set, it's Unicode.
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
enables setting the filename not via the MIME filename, but via a
separate field called "name".
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
enables setting the filename not via the MIME filename, but via a
separate field called "name".
Thus we need to do this ridiculous workaround. Mypy doesn't like it
either, thus the ``# type: ignore`` below.
Thus we need to do this ridiculous workaround. Mypy doesn't like it
either, thus the ``# type: ignore`` below.
Source for idea:
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
"""
@property # type: ignore
def filename(self):
if self.name == "file" and not self._mime_filename:
# We use the file field to upload files, see directory.py's
# _POST_upload. Lack of _mime_filename means we need to trick
# FieldStorage into thinking there is a filename so it'll
# return bytes.
return "unknown-filename"
return self._mime_filename
Source for idea:
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
"""
@property # type: ignore
def filename(self):
if self.name == "file" and not self._mime_filename:
# We use the file field to upload files, see directory.py's
# _POST_upload. Lack of _mime_filename means we need to trick
# FieldStorage into thinking there is a filename so it'll
# return bytes.
return "unknown-filename"
return self._mime_filename
@filename.setter
def filename(self, value):
self._mime_filename = value
@filename.setter
def filename(self, value):
self._mime_filename = value
class TahoeLAFSRequest(Request, object):
@ -180,12 +171,7 @@ def _logFormatter(logDateTime, request):
queryargs = b""
else:
path, queryargs = x
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these too.
if queryargs.startswith(b"uri="):
queryargs = b"uri=[CENSORED]"
queryargs = b"?" + queryargs
queryargs = b"?" + censor(queryargs)
if path.startswith(b"/uri/"):
path = b"/uri/[CENSORED]"
elif path.startswith(b"/file/"):
@ -207,6 +193,30 @@ def _logFormatter(logDateTime, request):
)
def censor(queryargs: bytes) -> bytes:
"""
Replace potentially sensitive values in query arguments with a
constant string.
"""
args = parse_qsl(queryargs.decode("ascii"), keep_blank_values=True, encoding="utf8")
result = []
for k, v in args:
if k == "uri":
# there is a form handler which redirects POST /uri?uri=FOO into
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
# sure we censor these.
v = "[CENSORED]"
elif k == "private-key":
# Likewise, sometimes a private key is supplied with mutable
# creation.
v = "[CENSORED]"
result.append((k, v))
# Customize safe to try to leave our markers intact.
return urlencode(result, safe="[]").encode("ascii")
class TahoeLAFSSite(Site, object):
"""
The HTTP protocol factory used by Tahoe-LAFS.

4
tests.nix
View File

@ -5,7 +5,7 @@ in
{ pkgsVersion ? "nixpkgs-21.11"
, pkgs ? import sources.${pkgsVersion} { }
, pypiData ? sources.pypi-deps-db
, pythonVersion ? "python37"
, pythonVersion ? "python39"
, mach-nix ? import sources.mach-nix {
inherit pkgs pypiData;
python = pythonVersion;
@ -21,7 +21,7 @@ let
inherit pkgs;
lib = pkgs.lib;
};
tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test;
tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test;
# Get the Tahoe-LAFS package itself. This does not include test
# requirements and we don't ask for test requirements so that we can just

6
tox.ini
View File

@ -7,11 +7,9 @@
# the tox-gh-actions package.
[gh-actions]
python =
3.7: py37-coverage
3.8: py38-coverage
3.9: py39-coverage
3.10: py310-coverage
pypy-3.7: pypy37
pypy-3.8: pypy38
pypy-3.9: pypy39
@ -19,7 +17,7 @@ python =
twisted = 1
[tox]
envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration
envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration
minversion = 2.4
[testenv]
@ -49,8 +47,6 @@ deps =
# regressions in new releases of this package that cause us the kind of
# suffering we're trying to avoid with the above pins.
certifi
# VCS hooks support
py37,!coverage: pre-commit
# We add usedevelop=False because testing against a true installation gives
# more useful results.