diff --git a/.circleci/config.yml b/.circleci/config.yml index afa3fafa1..29b55ad5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,9 @@ workflows: - "build-porting-depgraph": <<: *DOCKERHUB_CONTEXT + - "typechecks": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -475,6 +478,18 @@ jobs: . /tmp/venv/bin/activate ./misc/python3/depgraph.sh + typechecks: + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/ubuntu:18.04-py3" + + steps: + - "checkout" + - run: + name: "Validate Types" + command: | + /tmp/venv/bin/tox -e typechecks + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index c8f5093f1..b59385aa4 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -17,4 +17,4 @@ Examples of contributions include: * `Patch reviews `_ Before authoring or reviewing a patch, -please familiarize yourself with the `coding standard `_. +please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5049104..ee36833ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,37 @@ jobs: with: args: install vcpython27 + # See https://github.com/actions/checkout. A fetch-depth of 0 + # fetches all tags and branches. - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + # To use pip caching with GitHub Actions in an OS-independent + # manner, we need `pip cache dir` command, which became + # available since pip v20.1+. At the time of writing this, + # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and + # windows-latest, and pip v20.3.1 for macos-latest. + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + # See https://github.com/actions/cache + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade codecov tox setuptools @@ -103,15 +123,27 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade tox @@ -155,15 +187,27 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade tox diff --git a/docs/README.md b/docs/README.txt similarity index 100% rename from docs/README.md rename to docs/README.txt diff --git a/docs/conf.py b/docs/conf.py index 34ddd1bd4..612c324a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ import os # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ['recommonmark'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -36,7 +36,7 @@ templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The encoding of source files. #source_encoding = 'utf-8-sig' diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 000000000..15e1b6432 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../.github/CONTRIBUTING.rst diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index 417109213..77ce11974 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -2032,10 +2032,11 @@ potential for surprises when the file store structure is changed. Tahoe-LAFS provides a mutable file store, but the ways that the store can change are limited. The only things that can change are: - * the mapping from child names to child objects inside mutable directories - (by adding a new child, removing an existing child, or changing an - existing child to point to a different object) - * the contents of mutable files + +* the mapping from child names to child objects inside mutable directories + (by adding a new child, removing an existing child, or changing an + existing child to point to a different object) +* the contents of mutable files Obviously if you query for information about the file store and then act to change it (such as by getting a listing of the contents of a mutable diff --git a/docs/index.rst b/docs/index.rst index 3d0a41302..60a3aa5d4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,8 +23,9 @@ Contents: frontends/download-status known_issues - ../.github/CONTRIBUTING + contributing CODE_OF_CONDUCT + release-checklist servers helper diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index be32aea6c..52d7d9344 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -40,23 +40,31 @@ Create Branch and Apply Updates - Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`) - run `tox -e news` to produce a new NEWS.txt file (this does a commit) - create the news for the release + - newsfragments/.minor - commit it + - manually fix NEWS.txt + - proper title for latest release ("Release 1.15.0" instead of "Release ...post1432") - double-check date (maybe release will be in the future) - spot-check the release notes (these come from the newsfragments files though so don't do heavy editing) - commit these changes + - update "relnotes.txt" + - update all mentions of 1.14.0 -> 1.15.0 - update "previous release" statement and date - summarize major changes - commit it + - update "CREDITS" + - are there any new contributors in this release? - one way: git log release-1.14.0.. | grep Author | sort | uniq - commit it + - update "docs/known_issues.rst" if appropriate - update "docs/INSTALL.rst" references to the new release - Push the branch to github @@ -82,25 +90,36 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release + - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0 - (replace the key-id above with your own) + - build all code locally - these should all pass: + - tox -e py27,codechecks,docs,integration + - these can fail (ideally they should not of course): + - tox -e deprecations,upcoming-deprecations + - build tarballs + - tox -e tarballs - confirm it at least exists: - ls dist/ | grep 1.15.0rc0 + - inspect and test the tarballs + - install each in a fresh virtualenv - run `tahoe` command + - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.zip + + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip Privileged Contributor @@ -129,6 +148,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` https://tahoe-lafs.org/downloads/ on the Web. - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads - the following developers have access to do this: + - exarkun - meejah - warner @@ -137,8 +157,9 @@ For the actual release, the tarball and signature files need to be uploaded to PyPI as well. - how to do this? -- (original guide says only "twine upload dist/*") +- (original guide says only `twine upload dist/*`) - the following developers have access to do this: + - warner - exarkun (partial?) - meejah (partial?) diff --git a/integration/conftest.py b/integration/conftest.py index 4ae22deee..533cbdb67 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -40,7 +40,8 @@ from util import ( TahoeProcess, cli, _run_node, - generate_ssh_key + generate_ssh_key, + block_with_timeout, ) @@ -156,7 +157,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): ) print("Waiting for flogtool to complete") try: - pytest_twisted.blockon(flog_protocol.done) + block_with_timeout(flog_protocol.done, reactor) except ProcessTerminated as e: print("flogtool exited unexpectedly: {}".format(str(e))) print("Flogtool completed") @@ -297,7 +298,7 @@ log_gatherer.furl = {log_furl} def cleanup(): try: transport.signalProcess('TERM') - pytest_twisted.blockon(protocol.exited) + block_with_timeout(protocol.exited, reactor) except ProcessExitedAlready: pass request.addfinalizer(cleanup) @@ -351,6 +352,9 @@ def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, requ reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", web_port="tcp:9980:interface=localhost", storage=False, + # We're going to kill this ourselves, so no need for finalizer to + # do it: + finalize=False, ) ) await_client_ready(process) @@ -533,7 +537,13 @@ def tor_network(reactor, temp_dir, chutney, request): path=join(chutney_dir), env=env, ) - pytest_twisted.blockon(proto.done) + try: + block_with_timeout(proto.done, reactor) + except ProcessTerminated: + # If this doesn't exit cleanly, that's fine, that shouldn't fail + # the test suite. + pass + request.addfinalizer(cleanup) return chut diff --git a/integration/test_sftp.py b/integration/test_sftp.py index f1cf92eab..6171c7413 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -1,6 +1,13 @@ """ It's possible to create/rename/delete files and directories in Tahoe-LAFS using SFTP. + +These tests use Paramiko, rather than Twisted's Conch, because: + + 1. It's a different implementation, so we're not testing Conch against + itself. + + 2. Its API is much simpler to use. """ from __future__ import unicode_literals @@ -23,7 +30,7 @@ from paramiko.rsakey import RSAKey import pytest -from .util import generate_ssh_key +from .util import generate_ssh_key, run_in_thread def connect_sftp(connect_args={"username": "alice", "password": "password"}): @@ -50,6 +57,7 @@ def connect_sftp(connect_args={"username": "alice", "password": "password"}): return sftp +@run_in_thread def test_bad_account_password_ssh_key(alice, tmpdir): """ Can't login with unknown username, wrong password, or wrong SSH pub key. @@ -79,6 +87,7 @@ def test_bad_account_password_ssh_key(alice, tmpdir): }) +@run_in_thread def test_ssh_key_auth(alice): """It's possible to login authenticating with SSH public key.""" key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) @@ -88,20 +97,21 @@ def test_ssh_key_auth(alice): assert sftp.listdir() == [] +@run_in_thread def test_read_write_files(alice): """It's possible to upload and download files.""" sftp = connect_sftp() - f = sftp.file("myfile", "wb") - f.write(b"abc") - f.write(b"def") - f.close() - f = sftp.file("myfile", "rb") - assert f.read(4) == b"abcd" - assert f.read(2) == b"ef" - assert f.read(1) == b"" - f.close() + with sftp.file("myfile", "wb") as f: + f.write(b"abc") + f.write(b"def") + + with sftp.file("myfile", "rb") as f: + assert f.read(4) == b"abcd" + assert f.read(2) == b"ef" + assert f.read(1) == b"" +@run_in_thread def test_directories(alice): """ It's possible to create, list directories, and create and remove files in @@ -135,6 +145,7 @@ def test_directories(alice): assert sftp.listdir() == [] +@run_in_thread def test_rename(alice): """Directories and files can be renamed.""" sftp = connect_sftp() diff --git a/integration/test_web.py b/integration/test_web.py index fe2137ff3..aab11412f 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -127,12 +127,12 @@ def test_deep_stats(alice): dircap_uri, data={ u"t": u"upload", - u"when_done": u".", }, files={ u"file": FILE_CONTENTS, }, ) + resp.raise_for_status() # confirm the file is in the directory resp = requests.get( @@ -175,6 +175,7 @@ def test_deep_stats(alice): time.sleep(.5) +@util.run_in_thread def test_status(alice): """ confirm we get something sensible from /status and the various sub-types diff --git a/integration/util.py b/integration/util.py index 39ec36a38..256fd68c1 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ from os import mkdir, environ from os.path import exists, join from six.moves import StringIO from functools import partial -from subprocess import check_output, check_call +from subprocess import check_output from twisted.python.filepath import ( FilePath, @@ -13,10 +13,12 @@ from twisted.python.filepath import ( from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone +from twisted.internet.threads import deferToThread import requests from paramiko.rsakey import RSAKey +from boltons.funcutils import wraps from allmydata.util.configutil import ( get_config, @@ -28,6 +30,12 @@ from allmydata import client import pytest_twisted +def block_with_timeout(deferred, reactor, timeout=120): + """Block until Deferred has result, but timeout instead of waiting forever.""" + deferred.addTimeout(timeout, reactor) + return pytest_twisted.blockon(deferred) + + class _ProcessExitedProtocol(ProcessProtocol): """ Internal helper that .callback()s on self.done when the process @@ -126,11 +134,12 @@ def _cleanup_tahoe_process(tahoe_transport, exited): :return: After the process has exited. """ + from twisted.internet import reactor try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') print("signaled, blocking on exit") - pytest_twisted.blockon(exited) + block_with_timeout(exited, reactor) print("exited, goodbye") except ProcessExitedAlready: pass @@ -186,7 +195,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text): +def _run_node(reactor, node_dir, request, magic_text, finalize=True): """ Run a tahoe process from its node_dir. @@ -210,7 +219,8 @@ def _run_node(reactor, node_dir, request, magic_text): ) transport.exited = protocol.exited - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + if finalize: + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern @@ -229,7 +239,8 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam magic_text=None, needed=2, happy=3, - total=4): + total=4, + finalize=True): """ Helper to create a single node, run it and return the instance spawnProcess returned (ITransport) @@ -256,7 +267,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--helper', ] if not storage: - args.append('--no-storage') + args.append('--no-storage') args.append(node_dir) _tahoe_runner_optional_coverage(done_proto, reactor, request, args) @@ -277,7 +288,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam d = Deferred() d.callback(None) d.addCallback(lambda _: created_d) - d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text)) + d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text, finalize=finalize)) return d @@ -516,3 +527,28 @@ def generate_ssh_key(path): key.write_private_key_file(path) with open(path + ".pub", "wb") as f: f.write(b"%s %s" % (key.get_name(), key.get_base64())) + + +def run_in_thread(f): + """Decorator for integration tests that runs code in a thread. + + Because we're using pytest_twisted, tests that rely on the reactor are + expected to return a Deferred and use async APIs so the reactor can run. + + In the case of the integration test suite, it launches nodes in the + background using Twisted APIs. The nodes stdout and stderr is read via + Twisted code. If the reactor doesn't run, reads don't happen, and + eventually the buffers fill up, and the nodes block when they try to flush + logs. + + We can switch to Twisted APIs (treq instead of requests etc.), but + sometimes it's easier or expedient to just have a blocking test. So this + decorator allows you to run the test in a thread, and the reactor can keep + running in the main thread. + + See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug. + """ + @wraps(f) + def test(*args, **kwargs): + return deferToThread(lambda: f(*args, **kwargs)) + return test diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..01cbb57a8 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +ignore_missing_imports = True +plugins=mypy_zope:plugin diff --git a/newsfragments/2920.minor b/newsfragments/2920.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature new file mode 100644 index 000000000..d30a91679 --- /dev/null +++ b/newsfragments/3399.feature @@ -0,0 +1 @@ +Added 'typechecks' environment for tox running mypy and performing static typechecks. diff --git a/newsfragments/3536.minor b/newsfragments/3536.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3576.minor b/newsfragments/3576.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3584.bugfix b/newsfragments/3584.bugfix index 73650f40b..faf57713b 100644 --- a/newsfragments/3584.bugfix +++ b/newsfragments/3584.bugfix @@ -1 +1 @@ -SFTP public key auth likely works better, and SFTP in general was broken in the prerelease. \ No newline at end of file +SFTP public key auth likely works more consistently, and SFTP in general was previously broken. \ No newline at end of file diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3590.bugfix b/newsfragments/3590.bugfix new file mode 100644 index 000000000..aa504a5e3 --- /dev/null +++ b/newsfragments/3590.bugfix @@ -0,0 +1 @@ +Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. \ No newline at end of file diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3599.minor b/newsfragments/3599.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 952a921bc..5dc68d367 100644 --- a/setup.py +++ b/setup.py @@ -396,6 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "junitxml", "tenacity", "paramiko", + "pytest-timeout", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..3157c8c80 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,9 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +26,9 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a4baab4b6..19345959e 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -57,6 +57,10 @@ class CRSEncoder(object): return defer.succeed((shares, desired_share_ids)) + def encode_proposal(self, data, desired_share_ids=None): + raise NotImplementedError() + + @implementer(ICodecDecoder) class CRSDecoder(object): diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index e8b80b9ad..6871b94c7 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -568,7 +568,7 @@ class DirectoryNode(object): d = self.get_child_and_metadata(childnamex) return d - def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True): + def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True): precondition(isinstance(writecap, (bytes, type(None))), writecap) precondition(isinstance(readcap, (bytes, type(None))), readcap) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index 51b161dda..b737e0473 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1269,7 +1269,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata), level=NOISY) - _assert((isinstance(userpath, bytes) and isinstance(childname, (str, type(None)) (metadata is None or 'no-write' in metadata)), + _assert((isinstance(userpath, bytes) and isinstance(childname, (str, type(None))) and + (metadata is None or 'no-write' in metadata)), userpath=userpath, childname=childname, metadata=metadata) writing = (flags & (FXF_WRITE | FXF_CREAT)) != 0 diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 68db478f3..6ed5571b9 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -19,7 +19,7 @@ from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI -@implementer(IImmutableFileNode, ICheckable) + class _ImmutableFileNodeBase(object): def get_write_uri(self): @@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object): return not self == other +@implementer(IImmutableFileNode, ICheckable) class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index d574b980d..2d2c5c1f5 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object): @implementer(interfaces.RICHKUploadHelper) -class CHKUploadHelper(Referenceable, upload.CHKUploader): +class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warner/foolscap#78 """I am the helper-server -side counterpart to AssistedUploader. I handle peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. @@ -499,10 +499,13 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561 + def set_upload_status(self, upload_status): + raise NotImplementedError @implementer(interfaces.RIHelper, interfaces.IStatsProducer) -class Helper(Referenceable): +class Helper(Referenceable): # type: ignore # warner/foolscap#78 """ :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which have been started but not finished, a mapping from storage index to the diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6dae825ac..46e01184f 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -13,19 +13,30 @@ if PY2: from past.builtins import long, unicode from six import ensure_str +try: + from typing import List +except ImportError: + pass + import os, time, weakref, itertools + +import attr + from zope.interface import implementer from twisted.python import failure from twisted.internet import defer from twisted.application import service -from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually +from foolscap.api import Referenceable, Copyable, RemoteCopy from allmydata.crypto import aes from allmydata.util.hashutil import file_renewal_secret_hash, \ file_cancel_secret_hash, bucket_renewal_secret_hash, \ bucket_cancel_secret_hash, plaintext_hasher, \ storage_index_hash, plaintext_segment_hasher, convergence_hasher -from allmydata.util.deferredutil import timeout_call +from allmydata.util.deferredutil import ( + timeout_call, + until, +) from allmydata import hashtree, uri from allmydata.storage.server import si_b2a from allmydata.immutable import encode @@ -386,6 +397,9 @@ class PeerSelector(object): ) return self.happiness_mappings + def add_peers(self, peerids=None): + raise NotImplementedError + class _QueryStatistics(object): @@ -897,13 +911,45 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): raise UploadUnhappinessError(msg) +@attr.s +class _Accum(object): + """ + Accumulate up to some known amount of ciphertext. + + :ivar remaining: The number of bytes still expected. + :ivar ciphertext: The bytes accumulated so far. + """ + remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int + ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes] + + def extend(self, + size, # type: int + ciphertext, # type: List[bytes] + ): + """ + Accumulate some more ciphertext. + + :param size: The amount of data the new ciphertext represents towards + the goal. This may be more than the actual size of the given + ciphertext if the source has run out of data. + + :param ciphertext: The new ciphertext to accumulate. + """ + self.remaining -= size + self.ciphertext.extend(ciphertext) + + @implementer(IEncryptedUploadable) class EncryptAnUploadable(object): """This is a wrapper that takes an IUploadable and provides IEncryptedUploadable.""" CHUNKSIZE = 50*1024 - def __init__(self, original, log_parent=None, progress=None): + def __init__(self, original, log_parent=None, progress=None, chunk_size=None): + """ + :param chunk_size: The number of bytes to read from the uploadable at a + time, or None for some default. + """ precondition(original.default_params_set, "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,)) self.original = IUploadable(original) @@ -917,6 +963,8 @@ class EncryptAnUploadable(object): self._ciphertext_bytes_read = 0 self._status = None self._progress = progress + if chunk_size is not None: + self.CHUNKSIZE = chunk_size def set_upload_status(self, upload_status): self._status = IUploadStatus(upload_status) @@ -1023,47 +1071,53 @@ class EncryptAnUploadable(object): # and size d.addCallback(lambda ignored: self.get_size()) d.addCallback(lambda ignored: self._get_encryptor()) - # then fetch and encrypt the plaintext. The unusual structure here - # (passing a Deferred *into* a function) is needed to avoid - # overflowing the stack: Deferreds don't optimize out tail recursion. - # We also pass in a list, to which _read_encrypted will append - # ciphertext. - ciphertext = [] - d2 = defer.Deferred() - d.addCallback(lambda ignored: - self._read_encrypted(length, ciphertext, hash_only, d2)) - d.addCallback(lambda ignored: d2) + + accum = _Accum(length) + + def action(): + """ + Read some bytes into the accumulator. + """ + return self._read_encrypted(accum, hash_only) + + def condition(): + """ + Check to see if the accumulator has all the data. + """ + return accum.remaining == 0 + + d.addCallback(lambda ignored: until(action, condition)) + d.addCallback(lambda ignored: accum.ciphertext) return d - def _read_encrypted(self, remaining, ciphertext, hash_only, fire_when_done): - if not remaining: - fire_when_done.callback(ciphertext) - return None + def _read_encrypted(self, + ciphertext_accum, # type: _Accum + hash_only, # type: bool + ): + # type: (...) -> defer.Deferred + """ + Read the next chunk of plaintext, encrypt it, and extend the accumulator + with the resulting ciphertext. + """ # tolerate large length= values without consuming a lot of RAM by # reading just a chunk (say 50kB) at a time. This only really matters # when hash_only==True (i.e. resuming an interrupted upload), since # that's the case where we will be skipping over a lot of data. - size = min(remaining, self.CHUNKSIZE) - remaining = remaining - size + size = min(ciphertext_accum.remaining, self.CHUNKSIZE) + # read a chunk of plaintext.. d = defer.maybeDeferred(self.original.read, size) - # N.B.: if read() is synchronous, then since everything else is - # actually synchronous too, we'd blow the stack unless we stall for a - # tick. Once you accept a Deferred from IUploadable.read(), you must - # be prepared to have it fire immediately too. - d.addCallback(fireEventually) def _good(plaintext): # and encrypt it.. # o/' over the fields we go, hashing all the way, sHA! sHA! sHA! o/' ct = self._hash_and_encrypt_plaintext(plaintext, hash_only) - ciphertext.extend(ct) - self._read_encrypted(remaining, ciphertext, hash_only, - fire_when_done) - def _err(why): - fire_when_done.errback(why) + # Intentionally tell the accumulator about the expected size, not + # the actual size. If we run out of data we still want remaining + # to drop otherwise it will never reach 0 and the loop will never + # end. + ciphertext_accum.extend(size, ct) d.addCallback(_good) - d.addErrback(_err) - return None + return d def _hash_and_encrypt_plaintext(self, data, hash_only): assert isinstance(data, (tuple, list)), type(data) @@ -1424,7 +1478,7 @@ class LiteralUploader(object): return self._status @implementer(RIEncryptedUploadable) -class RemoteEncryptedUploadable(Referenceable): +class RemoteEncryptedUploadable(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, encrypted_uploadable, upload_status): self._eu = IEncryptedUploadable(encrypted_uploadable) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 6d0938dd5..0dd5ddc83 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -681,7 +681,7 @@ class IURI(Interface): passing into init_from_string.""" -class IVerifierURI(Interface, IURI): +class IVerifierURI(IURI): def init_from_string(uri): """Accept a string (as created by my to_string() method) and populate this instance with its data. I am not normally called directly, @@ -748,7 +748,7 @@ class IProgress(Interface): "Current amount of progress (in percentage)" ) - def set_progress(self, value): + def set_progress(value): """ Sets the current amount of progress. @@ -756,7 +756,7 @@ class IProgress(Interface): set_progress_total. """ - def set_progress_total(self, value): + def set_progress_total(value): """ Sets the total amount of expected progress @@ -859,12 +859,6 @@ class IPeerSelector(Interface): peer selection begins. """ - def confirm_share_allocation(peerid, shnum): - """ - Confirm that an allocated peer=>share pairing has been - successfully established. - """ - def add_peers(peerids=set): """ Update my internal state to include the peers in peerids as @@ -1824,11 +1818,6 @@ class IEncoder(Interface): willing to receive data. """ - def set_size(size): - """Specify the number of bytes that will be encoded. This must be - peformed before get_serialized_params() can be called. - """ - def set_encrypted_uploadable(u): """Provide a source of encrypted upload data. 'u' must implement IEncryptedUploadable. diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 72d68acec..07f8a5f7a 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -178,9 +178,9 @@ class IntroducerClient(service.Service, Referenceable): kwargs["facility"] = "tahoe.introducer.client" return log.msg(*args, **kwargs) - def subscribe_to(self, service_name, cb, *args, **kwargs): + def subscribe_to(self, service_name, callback, *args, **kwargs): obs = self._local_subscribers.setdefault(service_name, ObserverList()) - obs.subscribe(lambda key_s, ann: cb(key_s, ann, *args, **kwargs)) + obs.subscribe(lambda key_s, ann: callback(key_s, ann, *args, **kwargs)) self._maybe_subscribe() for index,(ann,key_s,when) in list(self._inbound_announcements.items()): precondition(isinstance(key_s, bytes), key_s) diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 9f08f1943..24fd3945f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -73,7 +73,7 @@ class IIntroducerClient(Interface): publish their services to the rest of the world, and I help them learn about services available on other nodes.""" - def publish(service_name, ann, signing_key=None): + def publish(service_name, ann, signing_key): """Publish the given announcement dictionary (which must be JSON-serializable), plus some additional keys, to the world. @@ -83,8 +83,7 @@ class IIntroducerClient(Interface): the signing_key, if present, otherwise it is derived from the 'anonymous-storage-FURL' key. - If signing_key= is set to an instance of SigningKey, it will be - used to sign the announcement.""" + signing_key (a SigningKey) will be used to sign the announcement.""" def subscribe_to(service_name, callback, *args, **kwargs): """Call this if you will eventually want to use services with the diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 237c30315..339c5a0ac 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -15,6 +15,12 @@ from past.builtins import long from six import ensure_text import time, os.path, textwrap + +try: + from typing import Any, Dict, Union +except ImportError: + pass + from zope.interface import implementer from twisted.application import service from twisted.internet import defer @@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable): name = "introducer" # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 - VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, + # TODO: reconcile bytes/str for keys + VERSION = { + #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } + } # type: Dict[Union[bytes, str], Any] def __init__(self): service.MultiService.__init__(self) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 5afc84dec..39e8b76be 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap): + def upload(self, new_contents, servermap, progress=None): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of @@ -951,7 +951,7 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): + def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. @@ -1205,3 +1205,7 @@ class MutableFileVersion(object): self._servermap, mode=mode) return u.update() + + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562 + def get_servermap(self): + raise NotImplementedError diff --git a/src/allmydata/node.py b/src/allmydata/node.py index e08c07508..2f340f860 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -23,6 +23,11 @@ from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn +try: + from typing import Union +except ImportError: + pass + import attr # On Python 2 this will be the backported package. @@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files): raise e +def ensure_text_and_abspath_expanduser_unicode(basedir): + # type: (Union[bytes, str]) -> str + return abspath_expanduser_unicode(ensure_text(basedir)) + + @attr.s class _Config(object): """ @@ -300,8 +310,8 @@ class _Config(object): config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser)) portnum_fname = attr.ib() _basedir = attr.ib( - converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)), - ) + converter=ensure_text_and_abspath_expanduser_unicode, + ) # type: str config_path = attr.ib( validator=attr.validators.optional( attr.validators.instance_of(FilePath), @@ -927,7 +937,6 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e472ffd8c..50dde9e43 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import BaseOptions @@ -79,8 +84,8 @@ def do_admin(options): subCommands = [ - ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] + ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), + ] # type: SubCommands dispatch = { "admin": do_admin, diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 379e1d212..e4cd8aa22 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,12 @@ from __future__ import print_function import os.path, re, fnmatch + +try: + from allmydata.scripts.types_ import SubCommands, Parameters +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions @@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] + ] # type: Parameters def postOptions(self): self["quiet"] = self.parent["quiet"] @@ -455,25 +461,25 @@ class DeepCheckOptions(FileStoreOptions): Optionally repair any problems found.""" subCommands = [ - ["mkdir", None, MakeDirectoryOptions, "Create a new directory."], - ["add-alias", None, AddAliasOptions, "Add a new alias cap."], - ["create-alias", None, CreateAliasOptions, "Create a new alias cap."], - ["list-aliases", None, ListAliasesOptions, "List all alias caps."], - ["ls", None, ListOptions, "List a directory."], - ["get", None, GetOptions, "Retrieve a file from the grid."], - ["put", None, PutOptions, "Upload a file into the grid."], - ["cp", None, CpOptions, "Copy one or more files or directories."], - ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["mv", None, MvOptions, "Move a file within the grid."], - ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], - ["backup", None, BackupOptions, "Make target dir look like local dir."], - ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."], - ["manifest", None, ManifestOptions, "List all files/directories in a subtree."], - ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."], - ["check", None, CheckOptions, "Check a single file or directory."], - ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."], - ["status", None, TahoeStatusCommand, "Various status information."], - ] + ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), + ("add-alias", None, AddAliasOptions, "Add a new alias cap."), + ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), + ("list-aliases", None, ListAliasesOptions, "List all alias caps."), + ("ls", None, ListOptions, "List a directory."), + ("get", None, GetOptions, "Retrieve a file from the grid."), + ("put", None, PutOptions, "Upload a file into the grid."), + ("cp", None, CpOptions, "Copy one or more files or directories."), + ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."), + ("mv", None, MvOptions, "Move a file within the grid."), + ("ln", None, LnOptions, "Make an additional link to an existing file or directory."), + ("backup", None, BackupOptions, "Make target dir look like local dir."), + ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."), + ("manifest", None, ManifestOptions, "List all files/directories in a subtree."), + ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."), + ("check", None, CheckOptions, "Check a single file or directory."), + ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), + ("status", None, TahoeStatusCommand, "Various status information."), + ] # type: SubCommands def mkdir(options): from allmydata.scripts import tahoe_mkdir diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 106dad3f2..d73344274 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,6 +4,12 @@ import os, sys, urllib, textwrap import codecs from os.path import join +try: + from typing import Optional + from .types_ import Parameters +except ImportError: + pass + from yaml import ( safe_dump, ) @@ -41,8 +47,8 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None - description_unwrapped = None + description = None # type: Optional[str] + description_unwrapped = None # type: Optional[str] def __str__(self): width = int(os.environ.get('COLUMNS', '80')) @@ -65,7 +71,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] + ] # type: Parameters def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -102,7 +108,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] + ] # type: Parameters # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index ac17cf445..0f507f518 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -3,6 +3,11 @@ from __future__ import print_function import os import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.internet import reactor, defer from twisted.python.usage import UsageError from twisted.python.filepath import ( @@ -492,10 +497,10 @@ def create_introducer(config): subCommands = [ - ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."], - ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."], - ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."], -] + ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), + ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), + ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), +] # type: SubCommands dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 451b1d661..550c37fde 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from future.utils import bchr # do not import any allmydata modules at this level. Do that from inside @@ -1053,8 +1058,8 @@ def do_debug(options): subCommands = [ - ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."], - ] + ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), + ] # type: SubCommands dispatch = { "debug": do_debug, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 1f993fda1..9a632a57d 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,11 @@ import os, sys from six.moves import StringIO import six +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, task, threads @@ -40,8 +45,8 @@ _control_node_dispatch = { } process_control_commands = [ - ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], -] + ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), +] # type: SubCommands class Options(usage.Options): @@ -98,7 +103,7 @@ class Options(usage.Options): create_dispatch = {} for module in (create_node,): - create_dispatch.update(module.dispatch) + create_dispatch.update(module.dispatch) # type: ignore def parse_options(argv, config=None): if not config: diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index dbc84d0ea..884536ec2 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -2,6 +2,11 @@ from __future__ import print_function import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, reactor @@ -103,7 +108,7 @@ def invite(options): subCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] +] # type: SubCommands dispatch = { "invite": invite, diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py new file mode 100644 index 000000000..3937cb803 --- /dev/null +++ b/src/allmydata/scripts/types_.py @@ -0,0 +1,12 @@ +from typing import List, Tuple, Type, Sequence, Any +from allmydata.scripts.common import BaseOptions + + +# Historically, subcommands were implemented as lists, but due to a +# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170), +# a Tuple is required. +SubCommand = Tuple[str, None, Type[BaseOptions], str] + +SubCommands = List[SubCommand] + +Parameters = List[Sequence[Any]] diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 41d81958b..91205a93c 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -23,7 +23,7 @@ from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 + POLL_INTERVAL = 60 # type: float def __init__(self): service.MultiService.__init__(self) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 24042c38b..f13f7cb99 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,7 +19,7 @@ import os, time, struct try: import cPickle as pickle except ImportError: - import pickle + import pickle # type: ignore from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 778c0ddf8..4b60d79f1 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -202,7 +202,7 @@ class ShareFile(object): @implementer(RIBucketWriter) -class BucketWriter(Referenceable): +class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): self.ss = ss @@ -301,7 +301,7 @@ class BucketWriter(Referenceable): @implementer(RIBucketReader) -class BucketReader(Referenceable): +class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8a8138f26..5f2ef3ac2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( + def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 4058ddf77..21576ea3a 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse +try: + from typing import Dict +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -49,13 +54,13 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} + last_stats = {} # type: Dict[str, float] while True: - stats = {} + stats = {} # type: Dict[str, float] for sf in statsfiles: for line in open(sf, "r").readlines(): - name, value = line.split(":") - value = int(value.strip()) + name, str_value = line.split(":") + value = int(str_value.strip()) if name not in stats: stats[name] = 0 stats[name] += float(value) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 6ec90eeae..268d77451 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -508,13 +508,13 @@ if __name__ == '__main__': mode = "upload" if len(sys.argv) > 1: mode = sys.argv[1] - if sys.maxint == 2147483647: + if sys.maxsize == 2147483647: bits = "32" - elif sys.maxint == 9223372036854775807: + elif sys.maxsize == 9223372036854775807: bits = "64" else: bits = "?" - print("%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)) + print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) # put the logfile and stats.out in _test_memory/ . These stick around. # put the nodes and other files in _test_memory/test/ . These are # removed each time we run. diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index f1dbf651d..fde92fb59 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -406,7 +406,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode(object): +class FakeCHKFileNode(object): # type: ignore # incomplete implementation """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -544,7 +544,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode(object): +class FakeMutableFileNode(object): # type: ignore # incomplete implementation """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index e1f04b864..cbea0dfcd 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -68,7 +68,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) +@implementer(IRemoteReference) # type: ignore # warner/foolscap#79 class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local @@ -213,9 +213,12 @@ class NoNetworkServer(object): return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version + def start_connecting(self, trigger_cb): + raise NotImplementedError + @implementer(IStorageBroker) -class NoNetworkStorageBroker(object): +class NoNetworkStorageBroker(object): # type: ignore # missing many methods def get_servers_for_psi(self, peer_selection_index): def _permuted(server): seed = server.get_permutation_seed() @@ -259,7 +262,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): +class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 """ Overrides all _Client networking functionality to do nothing. """ diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 4a1f84531..17ec89078 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -47,8 +47,9 @@ class RIDummy(RemoteInterface): """ - -@implementer(IFoolscapStoragePlugin) +# type ignored due to missing stubs for Twisted +# https://twistedmatrix.com/trac/ticket/9717 +@implementer(IFoolscapStoragePlugin) # type: ignore @attr.s class DummyStorage(object): name = attr.ib() @@ -107,7 +108,7 @@ class GetCounter(Resource, object): @implementer(RIDummy) @attr.s(frozen=True) -class DummyStorageServer(object): +class DummyStorageServer(object): # type: ignore # warner/foolscap#78 get_anonymous_storage_server = attr.ib() def remote_just_some_method(self): @@ -116,7 +117,7 @@ class DummyStorageServer(object): @implementer(IStorageServer) @attr.s -class DummyStorageClient(object): +class DummyStorageClient(object): # type: ignore # incomplete implementation get_rref = attr.ib() configuration = attr.ib() announcement = attr.ib() diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index a7042468a..f56ecd089 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): +class FakeServer(object): # type: ignore # incomplete implementation def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeCheckResults(object): +class FakeCheckResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, repair_attempted=False, diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 6ebc93556..2a155089f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -74,3 +74,58 @@ class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin d = defer.succeed(None) d.addBoth(self.wait_for_delayed_calls) return d + + +class UntilTests(unittest.TestCase): + """ + Tests for ``deferredutil.until``. + """ + def test_exception(self): + """ + If the action raises an exception, the ``Deferred`` returned by ``until`` + fires with a ``Failure``. + """ + self.assertFailure( + deferredutil.until(lambda: 1/0, lambda: True), + ZeroDivisionError, + ) + + def test_stops_on_condition(self): + """ + The action is called repeatedly until ``condition`` returns ``True``. + """ + calls = [] + def action(): + calls.append(None) + + def condition(): + return len(calls) == 3 + + self.assertIs( + self.successResultOf( + deferredutil.until(action, condition), + ), + None, + ) + self.assertEqual(3, len(calls)) + + def test_waits_for_deferred(self): + """ + If the action returns a ``Deferred`` then it is called again when the + ``Deferred`` fires. + """ + counter = [0] + r1 = defer.Deferred() + r2 = defer.Deferred() + results = [r1, r2] + def action(): + counter[0] += 1 + return results.pop(0) + + def condition(): + return False + + deferredutil.until(action, condition) + self.assertEqual([1], counter) + r1.callback(None) + self.assertEqual([2], counter) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 1c265492b..8e5e59b46 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1561,7 +1561,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): +class FakeMutableFile(object): # type: ignore # incomplete implementation counter = 0 def __init__(self, initial_contents=b""): data = self._get_initial_contents(initial_contents) @@ -1622,7 +1622,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents=b"", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): +class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 65c07135a..3faffbe0d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -19,6 +19,12 @@ from functools import ( ) import attr +try: + from typing import List + from allmydata.introducer.client import IntroducerClient +except ImportError: + pass + from twisted.internet import defer from twisted.trial import unittest from twisted.application import service @@ -125,7 +131,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] + introducer_clients = [] # type: List[IntroducerClient] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..e44fd5743 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -564,7 +564,7 @@ class TestMissingPorts(unittest.TestCase): config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( + test_listen_on_zero_with_host.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" ) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 84484f1cf..5c6a654c1 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -16,6 +16,7 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.test.check_load", "allmydata.windows.registry", + "allmydata.scripts.types_", } diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 80242f8a2..c1f0e83d6 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -44,7 +44,7 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = native_str( + test_finished_porting.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", ) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index ee02ae2d8..2214e4e5b 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -19,18 +19,15 @@ from twisted.python.failure import Failure from twisted.internet.error import ProcessDone, ProcessTerminated from allmydata.util import deferredutil -conch_interfaces = None -sftp = None -sftpd = None - try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd except ImportError as e: + conch_interfaces = sftp = sftpd = None # type: ignore conch_unavailable_reason = e else: - conch_unavailable_reason = None + conch_unavailable_reason = None # type: ignore from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 3a21dfd9e..8500d6bff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -105,7 +105,8 @@ from allmydata.interfaces import ( SOME_FURL = "pb://abcde@nowhere/fake" -class NativeStorageServerWithVersion(NativeStorageServer): + +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall @@ -569,7 +570,7 @@ class SpyEndpoint(object): return d -@implementer(IConnectionHintHandler) +@implementer(IConnectionHintHandler) # type: ignore # warner/foolscap#78 @attr.s class SpyHandler(object): """ diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 94d7575c3..07ede2074 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -14,6 +14,17 @@ if PY2: import os, shutil from io import BytesIO +from base64 import ( + b64encode, +) + +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + just, + integers, +) from twisted.trial import unittest from twisted.python.failure import Failure @@ -2029,6 +2040,91 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, f.close() return None + +class EncryptAnUploadableTests(unittest.TestCase): + """ + Tests for ``EncryptAnUploadable``. + """ + def test_same_length(self): + """ + ``EncryptAnUploadable.read_encrypted`` returns ciphertext of the same + length as the underlying plaintext. + """ + plaintext = b"hello world" + uploadable = upload.FileHandle(BytesIO(plaintext), None) + uploadable.set_default_encoding_parameters({ + # These values shouldn't matter. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + ciphertext = b"".join(self.successResultOf(encrypter.read_encrypted(1024, False))) + self.assertEqual(len(ciphertext), len(plaintext)) + + @given(just(b"hello world"), integers(min_value=0, max_value=len(b"hello world"))) + def test_known_result(self, plaintext, split_at): + """ + ``EncryptAnUploadable.read_encrypted`` returns a known-correct ciphertext + string for certain inputs. The ciphertext is independent of the read + sizes. + """ + convergence = b"\x42" * 16 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + # The convergence key is a function of k, n, and max_segment_size + # (among other things). The value for happy doesn't matter + # though. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + def read(n): + return b"".join(self.successResultOf(encrypter.read_encrypted(n, False))) + + # Read the string in one or two pieces to make sure underlying state + # is maintained properly. + first = read(split_at) + second = read(len(plaintext) - split_at) + third = read(1) + ciphertext = first + second + third + + self.assertEqual( + b"Jd2LHCRXozwrEJc=", + b64encode(ciphertext), + ) + + def test_large_read(self): + """ + ``EncryptAnUploadable.read_encrypted`` succeeds even when the requested + data length is much larger than the chunk size. + """ + convergence = b"\x42" * 16 + # 4kB of plaintext + plaintext = b"\xde\xad\xbe\xef" * 1024 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + # Make the chunk size very small so we don't have to operate on a huge + # amount of data to exercise the relevant codepath. + encrypter = upload.EncryptAnUploadable(uploadable, chunk_size=1) + d = encrypter.read_encrypted(len(plaintext), False) + ciphertext = self.successResultOf(d) + self.assertEqual( + list(map(len, ciphertext)), + # Chunk size was specified as 1 above so we will get the whole + # plaintext in one byte chunks. + [1] * len(plaintext), + ) + + # TODO: # upload with exactly 75 servers (shares_of_happiness) # have a download fail diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py deleted file mode 100644 index e666a4902..000000000 --- a/src/allmydata/test/test_websocket_logs.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from twisted.trial import unittest -from twisted.internet.defer import inlineCallbacks - -from eliot import log_call - -from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper - -from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol - - -class TestStreamingLogs(unittest.TestCase): - """ - Test websocket streaming of logs - """ - - def setUp(self): - self.reactor = MemoryReactorClockResolver() - self.pumper = create_pumper() - self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) - return self.pumper.start() - - def tearDown(self): - return self.pumper.stop() - - @inlineCallbacks - def test_one_log(self): - """ - write a single Eliot log and see it streamed via websocket - """ - - proto = yield self.agent.open( - transport_config=u"ws://localhost:1234/ws", - options={}, - ) - - messages = [] - def got_message(msg, is_binary=False): - messages.append(json.loads(msg)) - proto.on("message", got_message) - - @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): - pass - - do_a_thing() - - proto.transport.loseConnection() - yield proto.is_closed - - self.assertEqual(len(messages), 2) - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 929fba507..08d95bda9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import json from os.path import join @@ -213,7 +225,7 @@ class IntroducerRootTests(unittest.TestCase): resource = IntroducerRoot(introducer_node) response = json.loads( self.successResultOf( - render(resource, {"t": [b"json"]}), + render(resource, {b"t": [b"json"]}), ), ) self.assertEqual( diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 4895ed6f0..5d697f910 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.logs``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,19 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import json + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from eliot import log_call + +from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper + from testtools.matchers import ( Equals, ) @@ -37,6 +52,7 @@ from ..common import ( from ...web.logs import ( create_log_resources, + TokenAuthenticatedWebSocketServerProtocol, ) class StreamingEliotLogsTests(SyncTestCase): @@ -57,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase): self.client.get(b"http:///v1"), succeeded(has_response_code(Equals(OK))), ) + + +class TestStreamingLogs(unittest.TestCase): + """ + Test websocket streaming of logs + """ + + def setUp(self): + self.reactor = MemoryReactorClockResolver() + self.pumper = create_pumper() + self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) + return self.pumper.start() + + def tearDown(self): + return self.pumper.stop() + + @inlineCallbacks + def test_one_log(self): + """ + write a single Eliot log and see it streamed via websocket + """ + + proto = yield self.agent.open( + transport_config=u"ws://localhost:1234/ws", + options={}, + ) + + messages = [] + def got_message(msg, is_binary=False): + messages.append(json.loads(msg)) + proto.on("message", got_message) + + @log_call(action_type=u"test:cli:some-exciting-action") + def do_a_thing(): + pass + + do_a_thing() + + proto.transport.loseConnection() + yield proto.is_closed + + self.assertEqual(len(messages), 2) + self.assertEqual("started", messages[0]["action_status"]) + self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 27ddbcf78..b426b4d93 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.private``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from testtools.matchers import ( Equals, ) @@ -56,6 +62,7 @@ class PrivacyTests(SyncTestCase): return super(PrivacyTests, self).setUp() def _authorization(self, scheme, value): + value = str(value, "utf-8") return Headers({ u"authorization": [u"{} {}".format(scheme, value)], }) @@ -90,7 +97,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, u"foo bar"), + headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) @@ -103,7 +110,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, self.token), + headers=self._authorization(str(SCHEME, "utf-8"), self.token), ), # It's a made up URL so we don't get a 200, either, but a 404. succeeded(has_response_code(Equals(NOT_FOUND))), diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..ca3cc695d 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,6 +1,18 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time -from urllib import ( +from urllib.parse import ( quote, ) @@ -77,7 +89,7 @@ class RenderSlashUri(unittest.TestCase): ) self.assertEqual( response_body, - "Invalid capability", + b"Invalid capability", ) @@ -92,7 +104,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) class FakeClient(_Client): @@ -103,7 +115,7 @@ class RenderServiceRow(unittest.TestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server("test-srv", srv) + self.storage_broker.test_add_server(b"test-srv", srv) root = RootElement(FakeClient(), time.time) req = DummyRequest(b"") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e975464d3..2f000b7a1 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): +class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): +class FakeClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset @@ -4757,6 +4757,31 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi op_url = self.webish_url + "/operations/134?t=status&output=JSON" yield self.assertHTTPError(op_url, 404, "unknown/expired handle '134'") + @inlineCallbacks + def test_uri_redirect(self): + """URI redirects don't cause failure. + + Unit test reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3590 + """ + def req(method, path, **kwargs): + return treq.request(method, self.webish_url + path, persistent=False, + **kwargs) + + response = yield req("POST", "/uri?format=sdmf&t=mkdir") + dircap = yield response.content() + assert dircap.startswith('URI:DIR2:') + dircap_uri = "/uri/?uri={}&t=json".format(urllib.quote(dircap)) + + response = yield req( + "GET", + dircap_uri, + ) + self.assertEqual( + response.request.absoluteURI, + self.webish_url + "/uri/{}?t=json".format(urllib.quote(dircap))) + if response.code >= 400: + raise Error(response.code, response=response.content()) + def test_incident(self): d = self.POST("/report_incident", details="eek") def _done(res): diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index f79c88415..060696293 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -1,3 +1,13 @@ +"""Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 2c367cafe..51671b0ac 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -22,6 +22,11 @@ from past.builtins import unicode, long import re +try: + from typing import Type +except ImportError: + pass + from zope.interface import implementer from twisted.python.components import registerAdapter @@ -489,7 +494,7 @@ class MDMFVerifierURI(_BaseURI): return self -@implementer(IURI, IDirnodeURI) +@implementer(IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri @@ -536,7 +541,7 @@ class _DirectoryBaseURI(_BaseURI): return self._filenode_uri.get_storage_index() -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2:' @@ -555,7 +560,7 @@ class DirectoryURI(_DirectoryBaseURI): return ReadonlyDirectoryURI(self._filenode_uri.get_readonly()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-RO:' @@ -574,6 +579,7 @@ class ReadonlyDirectoryURI(_DirectoryBaseURI): return self +@implementer(IURI, IDirnodeURI) class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): def __init__(self, filenode_uri=None): if filenode_uri: @@ -611,7 +617,7 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): return None -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF:' @@ -633,7 +639,7 @@ class MDMFDirectoryURI(_DirectoryBaseURI): return MDMFDirectoryURIVerifier(self._filenode_uri.get_verify_cap()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-RO:' @@ -671,7 +677,7 @@ def wrap_dirnode_cap(filecap): raise AssertionError("cannot interpret as a directory cap: %s" % filecap.__class__) -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-Verifier:' @@ -696,12 +702,12 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): return self -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI + INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] def __init__(self, filenode_uri=None): if filenode_uri: diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 0aa7409c4..38d0f4d7e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -84,6 +84,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test.no_network", "allmydata.test.mutable.util", + "allmydata.unknown", "allmydata.uri", "allmydata.util._python3", "allmydata.util.abbreviate", @@ -114,6 +115,7 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.web.logs", "allmydata.webish", ] @@ -188,6 +190,10 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", + "allmydata.test.web.test_introducer", + "allmydata.test.web.test_logs", + "allmydata.test.web.test_private", + "allmydata.test.web.test_root", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 1d13f61e6..ed2a11ee4 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -15,7 +15,18 @@ if PY2: import time +try: + from typing import ( + Callable, + Any, + ) +except ImportError: + pass + from foolscap.api import eventually +from eliot.twisted import ( + inline_callbacks, +) from twisted.internet import defer, reactor, error from twisted.python.failure import Failure @@ -201,3 +212,22 @@ class WaitForDelayedCallsMixin(PollMixin): d.addErrback(log.err, "error while waiting for delayed calls") d.addBoth(lambda ign: res) return d + +@inline_callbacks +def until( + action, # type: Callable[[], defer.Deferred[Any]] + condition, # type: Callable[[], bool] +): + # type: (...) -> defer.Deferred[None] + """ + Run a Deferred-returning function until a condition is true. + + :param action: The action to run. + :param condition: The predicate signaling stop. + + :return: A Deferred that fires after the condition signals stop. + """ + while True: + yield action() + if condition(): + break diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index ea16c0d6a..e40e06180 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -311,7 +311,7 @@ def precondition_abspath(path): _getfullpathname = None try: - from nt import _getfullpathname + from nt import _getfullpathname # type: ignore except ImportError: pass diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 5d1716853..582bafe86 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -14,6 +14,12 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time + +try: + from typing import List +except ImportError: + pass + from twisted.internet import task class TimeoutError(Exception): @@ -23,7 +29,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] + _poll_should_ignore_these_errors = [] # type: List[Exception] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 3e9eb8379..cde3924fd 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -6,6 +6,11 @@ Can eventually be merged back into allmydata.web.common. from past.builtins import unicode +try: + from typing import Optional +except ImportError: + pass + from twisted.web import resource, http from allmydata.util import abbreviate @@ -55,7 +60,7 @@ class MultiFormatResource(resource.Resource, object): format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" - formatDefault = None + formatDefault = None # type: Optional[str] def render(self, req): """ diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 6ec558e82..280d6cc26 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource): self.introducer_node = introducer_node self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource - self.putChild("", self) + self.putChild(b"", self) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen))) def _create_element(self): """ @@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource): announcement_summary[service_name] += 1 res[u"announcement_summary"] = announcement_summary - return json.dumps(res, indent=1) + b"\n" + return (json.dumps(res, indent=1) + "\n").encode("utf-8") class IntroducerRootElement(Element): diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 0ba8b17e9..6f15a3ca9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -49,7 +52,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - self.sendMessage(json.dumps(message)) + encoded = json.dumps(message) + if isinstance(encoded, str): + # On Python 3 dumps() returns Unicode... + encoded = encoded.encode("utf-8") + self.sendMessage(encoded) def onOpen(self): """ diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index fea058405..405ca75e7 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,16 @@ class IToken(ICredentials): pass -@implementer(IToken) +# Workaround for Shoobx/mypy-zope#26, where without suitable +# stubs for twisted classes (ICredentials), IToken does not +# appear to be an Interface. The proper fix appears to be to +# create stubs for twisted +# (https://twistedmatrix.com/trac/ticket/9717). For now, +# bypassing the inline decorator syntax works around the issue. +_itoken_impl = implementer(IToken) + + +@_itoken_impl @attr.s class Token(object): proposed_token = attr.ib(type=bytes) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index fdc72ab71..9fb3ac9d3 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,6 +1,8 @@ +from future.utils import PY3 + import os import time -import urllib +from urllib.parse import quote as urlquote from hyperlink import DecodedURL, URL from pkg_resources import resource_filename @@ -9,7 +11,7 @@ from twisted.web import ( resource, static, ) -from twisted.web.util import redirectTo +from twisted.web.util import redirectTo, Redirect from twisted.python.filepath import FilePath from twisted.web.template import ( Element, @@ -81,7 +83,7 @@ class URIHandler(resource.Resource, object): # it seems Nevow was creating absolute URLs including # host/port whereas req.uri is absolute (but lacks host/port) redir_uri = URL.from_text(req.prePathURL().decode('utf8')) - redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) + redir_uri = redir_uri.child(urlquote(uri_arg)) # add back all the query args that AREN'T "?uri=" for k, values in req.args.items(): if k != b"uri": @@ -145,7 +147,7 @@ class URIHandler(resource.Resource, object): and creates and appropriate handler (depending on the kind of capability it was passed). """ - # this is in case a URI like "/uri/?cap=" is + # this is in case a URI like "/uri/?uri=" is # passed -- we re-direct to the non-trailing-slash version so # that there is just one valid URI for "uri" resource. if not name: @@ -153,7 +155,7 @@ class URIHandler(resource.Resource, object): u = u.replace( path=(s for s in u.path if s), # remove empty segments ) - return redirectTo(u.to_uri().to_text().encode('utf8'), req) + return Redirect(u.to_uri().to_text().encode('utf8')) try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) @@ -227,26 +229,26 @@ class Root(MultiFormatResource): self._client = client self._now_fn = now_fn - # Children need to be bytes; for now just doing these to make specific - # tests pass on Python 3, but eventually will do all them when this - # module is ported to Python 3 (if not earlier). self.putChild(b"uri", URIHandler(client)) - self.putChild("cap", URIHandler(client)) + self.putChild(b"cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource # hierarchy which is only accessible with the private per-node API # auth token. - self.putChild("private", create_private_tree(client.get_auth_token)) + self.putChild(b"private", create_private_tree(client.get_auth_token)) - self.putChild("file", FileHandler(client)) - self.putChild("named", FileHandler(client)) - self.putChild("status", status.Status(client.get_history())) - self.putChild("statistics", status.Statistics(client.stats_provider)) + self.putChild(b"file", FileHandler(client)) + self.putChild(b"named", FileHandler(client)) + self.putChild(b"status", status.Status(client.get_history())) + self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + child_path = filen + if PY3: + child_path = filen.encode("utf-8") + self.putChild(child_path, static.File(os.path.join(static_dir, filen))) - self.putChild("report_incident", IncidentReporter()) + self.putChild(b"report_incident", IncidentReporter()) @exception_to_child def getChild(self, path, request): diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e7f045b95..e98aa8a67 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -217,7 +217,12 @@ def initialize(): # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): - return re.sub(u'\\x7F[0-9a-fA-F]*\\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), s) + return re.sub( + u'\\x7F[0-9a-fA-F]*\\;', + # type ignored for 'unichr' (Python 2 only) + lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore + s, + ) try: argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)] diff --git a/tox.ini b/tox.ini index c61331885..915981e0c 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,pypy27 minversion = 2.4 [testenv] @@ -77,7 +77,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --coverage -v {posargs:integration} + py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report @@ -112,6 +112,16 @@ commands = # file. See pyproject.toml for legal values. python -m towncrier.check --pyproject towncrier.pyproject.toml + +[testenv:typechecks] +skip_install = True +deps = + mypy + git+https://github.com/Shoobx/mypy-zope + git+https://github.com/warner/foolscap +commands = mypy src + + [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" @@ -211,6 +221,7 @@ commands = deps = sphinx docutils==0.12 + recommonmark # normal install is not needed for docs, and slows things down skip_install = True commands =