diff --git a/.circleci/config.yml b/.circleci/config.yml index afa3fafa1..29b55ad5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,9 @@ workflows: - "build-porting-depgraph": <<: *DOCKERHUB_CONTEXT + - "typechecks": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -475,6 +478,18 @@ jobs: . /tmp/venv/bin/activate ./misc/python3/depgraph.sh + typechecks: + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/ubuntu:18.04-py3" + + steps: + - "checkout" + - run: + name: "Validate Types" + command: | + /tmp/venv/bin/tox -e typechecks + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5049104..ee36833ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,37 @@ jobs: with: args: install vcpython27 + # See https://github.com/actions/checkout. A fetch-depth of 0 + # fetches all tags and branches. - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + # To use pip caching with GitHub Actions in an OS-independent + # manner, we need `pip cache dir` command, which became + # available since pip v20.1+. At the time of writing this, + # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and + # windows-latest, and pip v20.3.1 for macos-latest. + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + # See https://github.com/actions/cache + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade codecov tox setuptools @@ -103,15 +123,27 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade tox @@ -155,15 +187,27 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Use pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Install Python packages run: | pip install --upgrade tox diff --git a/CREDITS b/CREDITS index 1394d87d8..07ac1e476 100644 --- a/CREDITS +++ b/CREDITS @@ -206,4 +206,9 @@ D: various bug-fixes and features N: Viktoriia Savchuk W: https://twitter.com/viktoriiasvchk -D: Developer community focused improvements on the README file. \ No newline at end of file +D: Developer community focused improvements on the README file. + +N: Lukas Pirl +E: tahoe@lukas-pirl.de +W: http://lukas-pirl.de +D: Buildslaves (Debian, Fedora, CentOS; 2016-2021) diff --git a/docs/about.rst b/docs/about.rst index 626792d6b..120abb079 100644 --- a/docs/about.rst +++ b/docs/about.rst @@ -67,12 +67,12 @@ Here's how it works: A "storage grid" is made up of a number of storage servers. A storage server has direct attached storage (typically one or more hard disks). A "gateway" communicates with storage nodes, and uses them to provide access to the -grid over protocols such as HTTP(S), SFTP or FTP. +grid over protocols such as HTTP(S) and SFTP. Note that you can find "client" used to refer to gateway nodes (which act as a client to storage servers), and also to processes or programs connecting to a gateway node and performing operations on the grid -- for example, a CLI -command, Web browser, SFTP client, or FTP client. +command, Web browser, or SFTP client. Users do not rely on storage servers to provide *confidentiality* nor *integrity* for their data -- instead all of the data is encrypted and diff --git a/docs/configuration.rst b/docs/configuration.rst index 2c0746ba2..93c9aa0f1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -81,7 +81,6 @@ Client/server nodes provide one or more of the following services: * web-API service * SFTP service -* FTP service * helper service * storage service. @@ -708,12 +707,12 @@ CLI file store, uploading/downloading files, and creating/running Tahoe nodes. See :doc:`frontends/CLI` for details. -SFTP, FTP +SFTP - Tahoe can also run both SFTP and FTP servers, and map a username/password + Tahoe can also run SFTP servers, and map a username/password pair to a top-level Tahoe directory. See :doc:`frontends/FTP-and-SFTP` - for instructions on configuring these services, and the ``[sftpd]`` and - ``[ftpd]`` sections of ``tahoe.cfg``. + for instructions on configuring this service, and the ``[sftpd]`` + section of ``tahoe.cfg``. Storage Server Configuration diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index dc348af34..ee6371812 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -1,22 +1,21 @@ .. -*- coding: utf-8-with-signature -*- -================================= -Tahoe-LAFS SFTP and FTP Frontends -================================= +======================== +Tahoe-LAFS SFTP Frontend +======================== -1. `SFTP/FTP Background`_ +1. `SFTP Background`_ 2. `Tahoe-LAFS Support`_ 3. `Creating an Account File`_ 4. `Running An Account Server (accounts.url)`_ 5. `Configuring SFTP Access`_ -6. `Configuring FTP Access`_ -7. `Dependencies`_ -8. `Immutable and Mutable Files`_ -9. `Known Issues`_ +6. `Dependencies`_ +7. `Immutable and Mutable Files`_ +8. `Known Issues`_ -SFTP/FTP Background -=================== +SFTP Background +=============== FTP is the venerable internet file-transfer protocol, first developed in 1971. The FTP server usually listens on port 21. A separate connection is @@ -33,20 +32,18 @@ Both FTP and SFTP were developed assuming a UNIX-like server, with accounts and passwords, octal file modes (user/group/other, read/write/execute), and ctime/mtime timestamps. -We recommend SFTP over FTP, because the protocol is better, and the server -implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below, -for details. +Previous versions of Tahoe-LAFS supported FTP, but now only the superior SFTP +frontend is supported. See `Known Issues`_, below, for details on the +limitations of SFTP. Tahoe-LAFS Support ================== All Tahoe-LAFS client nodes can run a frontend SFTP server, allowing regular SFTP clients (like ``/usr/bin/sftp``, the ``sshfs`` FUSE plugin, and many -others) to access the file store. They can also run an FTP server, so FTP -clients (like ``/usr/bin/ftp``, ``ncftp``, and others) can too. These -frontends sit at the same level as the web-API interface. +others) to access the file store. -Since Tahoe-LAFS does not use user accounts or passwords, the SFTP/FTP +Since Tahoe-LAFS does not use user accounts or passwords, the SFTP servers must be configured with a way to first authenticate a user (confirm that a prospective client has a legitimate claim to whatever authorities we might grant a particular user), and second to decide what directory cap @@ -173,39 +170,6 @@ clients and with the sshfs filesystem, see wiki:SftpFrontend_ .. _wiki:SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend -Configuring FTP Access -====================== - -To enable the FTP server with an accounts file, add the following lines to -the BASEDIR/tahoe.cfg file:: - - [ftpd] - enabled = true - port = tcp:8021:interface=127.0.0.1 - accounts.file = private/accounts - -The FTP server will listen on the given port number and on the loopback -interface only. The "accounts.file" pathname will be interpreted relative to -the node's BASEDIR. - -To enable the FTP server with an account server instead, provide the URL of -that server in an "accounts.url" directive:: - - [ftpd] - enabled = true - port = tcp:8021:interface=127.0.0.1 - accounts.url = https://example.com/login - -You can provide both accounts.file and accounts.url, although it probably -isn't very useful except for testing. - -FTP provides no security, and so your password or caps could be eavesdropped -if you connect to the FTP server remotely. The examples above include -":interface=127.0.0.1" in the "port" option, which causes the server to only -accept connections from localhost. - -Public key authentication is not supported for FTP. - Dependencies ============ @@ -216,7 +180,7 @@ separately: debian puts it in the "python-twisted-conch" package. Immutable and Mutable Files =========================== -All files created via SFTP (and FTP) are immutable files. However, files can +All files created via SFTP are immutable files. However, files can only be created in writeable directories, which allows the directory entry to be relinked to a different file. Normally, when the path of an immutable file is opened for writing by SFTP, the directory entry is relinked to another @@ -256,18 +220,3 @@ See also wiki:SftpFrontend_. .. _ticket #1059: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1059 .. _ticket #1089: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1089 - -Known Issues in the FTP Frontend --------------------------------- - -Mutable files are not supported by the FTP frontend (`ticket #680`_). - -Non-ASCII filenames are not supported by FTP (`ticket #682`_). - -The FTP frontend sometimes fails to report errors, for example if an upload -fails because it does meet the "servers of happiness" threshold (`ticket -#1081`_). - -.. _ticket #680: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/680 -.. _ticket #682: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/682 -.. _ticket #1081: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1081 diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index e00445fbf..77ce11974 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -2158,7 +2158,7 @@ When modifying the file, be careful to update it atomically, otherwise a request may arrive while the file is only halfway written, and the partial file may be incorrectly parsed. -The blacklist is applied to all access paths (including SFTP, FTP, and CLI +The blacklist is applied to all access paths (including SFTP and CLI operations), not just the web-API. The blacklist also applies to directories. If a directory is blacklisted, the gateway will refuse access to both that directory and any child files/directories underneath it, when accessed via diff --git a/docs/helper.rst b/docs/helper.rst index 0fcdf4601..55d302cac 100644 --- a/docs/helper.rst +++ b/docs/helper.rst @@ -122,7 +122,7 @@ Who should consider using a Helper? * clients who experience problems with TCP connection fairness: if other programs or machines in the same home are getting less than their fair share of upload bandwidth. If the connection is being shared fairly, then - a Tahoe upload that is happening at the same time as a single FTP upload + a Tahoe upload that is happening at the same time as a single SFTP upload should get half the bandwidth. * clients who have been given the helper.furl by someone who is running a Helper and is willing to let them use it diff --git a/docs/known_issues.rst b/docs/known_issues.rst index e040ffaf6..98bd1b35d 100644 --- a/docs/known_issues.rst +++ b/docs/known_issues.rst @@ -23,7 +23,7 @@ Known Issues in Tahoe-LAFS v1.10.3, released 30-Mar-2016 * `Disclosure of file through embedded hyperlinks or JavaScript in that file`_ * `Command-line arguments are leaked to other local users`_ * `Capabilities may be leaked to web browser phishing filter / "safe browsing" servers`_ - * `Known issues in the FTP and SFTP frontends`_ + * `Known issues in the SFTP frontend`_ * `Traffic analysis based on sizes of files/directories, storage indices, and timing`_ * `Privacy leak via Google Chart API link in map-update timing web page`_ @@ -213,8 +213,8 @@ To disable the filter in Chrome: ---- -Known issues in the FTP and SFTP frontends ------------------------------------------- +Known issues in the SFTP frontend +--------------------------------- These are documented in :doc:`frontends/FTP-and-SFTP` and on `the SftpFrontend page`_ on the wiki. diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index fabb69fc7..52d7d9344 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -116,10 +116,10 @@ they will need to evaluate which contributors' signatures they trust. - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.zip + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip Privileged Contributor diff --git a/docs/running.rst b/docs/running.rst index 6d82a97f2..82b0443f9 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -207,10 +207,10 @@ create a new directory and lose the capability to it, then you cannot access that directory ever again. -The SFTP and FTP frontends --------------------------- +The SFTP frontend +----------------- -You can access your Tahoe-LAFS grid via any SFTP_ or FTP_ client. See +You can access your Tahoe-LAFS grid via any SFTP_ client. See :doc:`frontends/FTP-and-SFTP` for how to set this up. On most Unix platforms, you can also use SFTP to plug Tahoe-LAFS into your computer's local filesystem via ``sshfs``, but see the `FAQ about performance @@ -220,7 +220,6 @@ The SftpFrontend_ page on the wiki has more information about using SFTP with Tahoe-LAFS. .. _SFTP: https://en.wikipedia.org/wiki/SSH_file_transfer_protocol -.. _FTP: https://en.wikipedia.org/wiki/File_Transfer_Protocol .. _FAQ about performance problems: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/FAQ#Q23_FUSE .. _SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..01cbb57a8 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +ignore_missing_imports = True +plugins=mypy_zope:plugin diff --git a/newsfragments/2920.minor b/newsfragments/2920.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature new file mode 100644 index 000000000..d30a91679 --- /dev/null +++ b/newsfragments/3399.feature @@ -0,0 +1 @@ +Added 'typechecks' environment for tox running mypy and performing static typechecks. diff --git a/newsfragments/3574.minor b/newsfragments/3574.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3576.minor b/newsfragments/3576.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3577.minor b/newsfragments/3577.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3582.minor b/newsfragments/3582.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3583.removed b/newsfragments/3583.removed new file mode 100644 index 000000000..a3fce48be --- /dev/null +++ b/newsfragments/3583.removed @@ -0,0 +1 @@ +FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead. \ No newline at end of file diff --git a/newsfragments/3587.minor b/newsfragments/3587.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3587.minor @@ -0,0 +1 @@ + diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index 0e5a43dba..205c00ae6 100644 --- a/setup.py +++ b/setup.py @@ -63,12 +63,8 @@ install_requires = [ # version of cryptography will *really* be installed. "cryptography >= 2.6", - # * We need Twisted 10.1.0 for the FTP frontend in order for - # Twisted's FTP server to support asynchronous close. # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server # rekeying bug - # * The FTP frontend depends on Twisted >= 11.1.0 for - # filepath.Permissions # * The SFTP frontend and manhole depend on the conch extra. However, we # can't explicitly declare that without an undesirable dependency on gmpy, # as explained in ticket #2740. diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..3157c8c80 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,9 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +26,9 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 1ee507117..b7e1d0956 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os diff --git a/src/allmydata/client.py b/src/allmydata/client.py index bd744fe6a..f5e603490 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -86,12 +86,6 @@ _client_config = configutil.ValidConfiguration( "shares.total", "storage.plugins", ), - "ftpd": ( - "accounts.file", - "accounts.url", - "enabled", - "port", - ), "storage": ( "debug_discard", "enabled", @@ -656,7 +650,6 @@ class _Client(node.Node, pollmixin.PollMixin): raise ValueError("config error: helper is enabled, but tub " "is not listening ('tub.port=' is empty)") self.init_helper() - self.init_ftp_server() self.init_sftp_server() # If the node sees an exit_trigger file, it will poll every second to see @@ -1032,18 +1025,6 @@ class _Client(node.Node, pollmixin.PollMixin): ) ws.setServiceParent(self) - def init_ftp_server(self): - if self.config.get_config("ftpd", "enabled", False, boolean=True): - accountfile = self.config.get_config("ftpd", "accounts.file", None) - if accountfile: - accountfile = self.config.get_config_path(accountfile) - accounturl = self.config.get_config("ftpd", "accounts.url", None) - ftp_portstr = self.config.get_config("ftpd", "port", "8021") - - from allmydata.frontends import ftpd - s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr) - s.setServiceParent(self) - def init_sftp_server(self): if self.config.get_config("sftpd", "enabled", False, boolean=True): accountfile = self.config.get_config("sftpd", "accounts.file", None) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a4baab4b6..19345959e 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -57,6 +57,10 @@ class CRSEncoder(object): return defer.succeed((shares, desired_share_ids)) + def encode_proposal(self, data, desired_share_ids=None): + raise NotImplementedError() + + @implementer(ICodecDecoder) class CRSDecoder(object): diff --git a/src/allmydata/deep_stats.py b/src/allmydata/deep_stats.py index c18adb5be..bfb43ebae 100644 --- a/src/allmydata/deep_stats.py +++ b/src/allmydata/deep_stats.py @@ -1,4 +1,15 @@ -"""Implementation of the deep stats class.""" +"""Implementation of the deep stats class. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import math @@ -13,7 +24,7 @@ from allmydata.util import mathutil class DeepStats(object): """Deep stats object. - Holds results of the deep-stats opetation. + Holds results of the deep-stats operation. Used for json generation in the API.""" # Json API version. @@ -121,7 +132,7 @@ class DeepStats(object): h[bucket] += 1 def get_results(self): - """Returns deep-stats resutls.""" + """Returns deep-stats results.""" stats = self.stats.copy() for key in self.histograms: h = self.histograms[key] diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index e8b80b9ad..6871b94c7 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -568,7 +568,7 @@ class DirectoryNode(object): d = self.get_child_and_metadata(childnamex) return d - def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True): + def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True): precondition(isinstance(writecap, (bytes, type(None))), writecap) precondition(isinstance(readcap, (bytes, type(None))), readcap) diff --git a/src/allmydata/frontends/ftpd.py b/src/allmydata/frontends/ftpd.py deleted file mode 100644 index 0b18df85b..000000000 --- a/src/allmydata/frontends/ftpd.py +++ /dev/null @@ -1,340 +0,0 @@ -from six import ensure_str - -from types import NoneType - -from zope.interface import implementer -from twisted.application import service, strports -from twisted.internet import defer -from twisted.internet.interfaces import IConsumer -from twisted.cred import portal -from twisted.python import filepath -from twisted.protocols import ftp - -from allmydata.interfaces import IDirectoryNode, ExistingChildError, \ - NoSuchChildError -from allmydata.immutable.upload import FileHandle -from allmydata.util.fileutil import EncryptedTemporaryFile -from allmydata.util.assertutil import precondition - -@implementer(ftp.IReadFile) -class ReadFile(object): - def __init__(self, node): - self.node = node - def send(self, consumer): - d = self.node.read(consumer) - return d # when consumed - -@implementer(IConsumer) -class FileWriter(object): - - def registerProducer(self, producer, streaming): - if not streaming: - raise NotImplementedError("Non-streaming producer not supported.") - # we write the data to a temporary file, since Tahoe can't do - # streaming upload yet. - self.f = EncryptedTemporaryFile() - return None - - def unregisterProducer(self): - # the upload actually happens in WriteFile.close() - pass - - def write(self, data): - self.f.write(data) - -@implementer(ftp.IWriteFile) -class WriteFile(object): - - def __init__(self, parent, childname, convergence): - self.parent = parent - self.childname = childname - self.convergence = convergence - - def receive(self): - self.c = FileWriter() - return defer.succeed(self.c) - - def close(self): - u = FileHandle(self.c.f, self.convergence) - d = self.parent.add_file(self.childname, u) - return d - - -class NoParentError(Exception): - pass - -# filepath.Permissions was added in Twisted-11.1.0, which we require. Twisted -# <15.0.0 expected an int, and only does '&' on it. Twisted >=15.0.0 expects -# a filepath.Permissions. This satisfies both. - -class IntishPermissions(filepath.Permissions): - def __init__(self, statModeInt): - self._tahoe_statModeInt = statModeInt - filepath.Permissions.__init__(self, statModeInt) - def __and__(self, other): - return self._tahoe_statModeInt & other - -@implementer(ftp.IFTPShell) -class Handler(object): - def __init__(self, client, rootnode, username, convergence): - self.client = client - self.root = rootnode - self.username = username - self.convergence = convergence - - def makeDirectory(self, path): - d = self._get_root(path) - d.addCallback(lambda root_and_path: - self._get_or_create_directories(root_and_path[0], root_and_path[1])) - return d - - def _get_or_create_directories(self, node, path): - if not IDirectoryNode.providedBy(node): - # unfortunately it is too late to provide the name of the - # blocking directory in the error message. - raise ftp.FileExistsError("cannot create directory because there " - "is a file in the way") - if not path: - return defer.succeed(node) - d = node.get(path[0]) - def _maybe_create(f): - f.trap(NoSuchChildError) - return node.create_subdirectory(path[0]) - d.addErrback(_maybe_create) - d.addCallback(self._get_or_create_directories, path[1:]) - return d - - def _get_parent(self, path): - # fire with (parentnode, childname) - path = [unicode(p) for p in path] - if not path: - raise NoParentError - childname = path[-1] - d = self._get_root(path) - def _got_root(root_and_path): - (root, path) = root_and_path - if not path: - raise NoParentError - return root.get_child_at_path(path[:-1]) - d.addCallback(_got_root) - def _got_parent(parent): - return (parent, childname) - d.addCallback(_got_parent) - return d - - def _remove_thing(self, path, must_be_directory=False, must_be_file=False): - d = defer.maybeDeferred(self._get_parent, path) - def _convert_error(f): - f.trap(NoParentError) - raise ftp.PermissionDeniedError("cannot delete root directory") - d.addErrback(_convert_error) - def _got_parent(parent_and_childname): - (parent, childname) = parent_and_childname - d = parent.get(childname) - def _got_child(child): - if must_be_directory and not IDirectoryNode.providedBy(child): - raise ftp.IsNotADirectoryError("rmdir called on a file") - if must_be_file and IDirectoryNode.providedBy(child): - raise ftp.IsADirectoryError("rmfile called on a directory") - return parent.delete(childname) - d.addCallback(_got_child) - d.addErrback(self._convert_error) - return d - d.addCallback(_got_parent) - return d - - def removeDirectory(self, path): - return self._remove_thing(path, must_be_directory=True) - - def removeFile(self, path): - return self._remove_thing(path, must_be_file=True) - - def rename(self, fromPath, toPath): - # the target directory must already exist - d = self._get_parent(fromPath) - def _got_from_parent(fromparent_and_childname): - (fromparent, childname) = fromparent_and_childname - d = self._get_parent(toPath) - d.addCallback(lambda toparent_and_tochildname: - fromparent.move_child_to(childname, - toparent_and_tochildname[0], toparent_and_tochildname[1], - overwrite=False)) - return d - d.addCallback(_got_from_parent) - d.addErrback(self._convert_error) - return d - - def access(self, path): - # we allow access to everything that exists. We are required to raise - # an error for paths that don't exist: FTP clients (at least ncftp) - # uses this to decide whether to mkdir or not. - d = self._get_node_and_metadata_for_path(path) - d.addErrback(self._convert_error) - d.addCallback(lambda res: None) - return d - - def _convert_error(self, f): - if f.check(NoSuchChildError): - childname = f.value.args[0].encode("utf-8") - msg = "'%s' doesn't exist" % childname - raise ftp.FileNotFoundError(msg) - if f.check(ExistingChildError): - msg = f.value.args[0].encode("utf-8") - raise ftp.FileExistsError(msg) - return f - - def _get_root(self, path): - # return (root, remaining_path) - path = [unicode(p) for p in path] - if path and path[0] == "uri": - d = defer.maybeDeferred(self.client.create_node_from_uri, - str(path[1])) - d.addCallback(lambda root: (root, path[2:])) - else: - d = defer.succeed((self.root,path)) - return d - - def _get_node_and_metadata_for_path(self, path): - d = self._get_root(path) - def _got_root(root_and_path): - (root,path) = root_and_path - if path: - return root.get_child_and_metadata_at_path(path) - else: - return (root,{}) - d.addCallback(_got_root) - return d - - def _populate_row(self, keys, childnode_and_metadata): - (childnode, metadata) = childnode_and_metadata - values = [] - isdir = bool(IDirectoryNode.providedBy(childnode)) - for key in keys: - if key == "size": - if isdir: - value = 0 - else: - value = childnode.get_size() or 0 - elif key == "directory": - value = isdir - elif key == "permissions": - # Twisted-14.0.2 (and earlier) expected an int, and used it - # in a rendering function that did (mode & NUMBER). - # Twisted-15.0.0 expects a - # twisted.python.filepath.Permissions , and calls its - # .shorthand() method. This provides both. - value = IntishPermissions(0o600) - elif key == "hardlinks": - value = 1 - elif key == "modified": - # follow sftpd convention (i.e. linkmotime in preference to mtime) - if "linkmotime" in metadata.get("tahoe", {}): - value = metadata["tahoe"]["linkmotime"] - else: - value = metadata.get("mtime", 0) - elif key == "owner": - value = self.username - elif key == "group": - value = self.username - else: - value = "??" - values.append(value) - return values - - def stat(self, path, keys=()): - # for files only, I think - d = self._get_node_and_metadata_for_path(path) - def _render(node_and_metadata): - (node, metadata) = node_and_metadata - assert not IDirectoryNode.providedBy(node) - return self._populate_row(keys, (node,metadata)) - d.addCallback(_render) - d.addErrback(self._convert_error) - return d - - def list(self, path, keys=()): - # the interface claims that path is a list of unicodes, but in - # practice it is not - d = self._get_node_and_metadata_for_path(path) - def _list(node_and_metadata): - (node, metadata) = node_and_metadata - if IDirectoryNode.providedBy(node): - return node.list() - return { path[-1]: (node, metadata) } # need last-edge metadata - d.addCallback(_list) - def _render(children): - results = [] - for (name, childnode) in children.iteritems(): - # the interface claims that the result should have a unicode - # object as the name, but it fails unless you give it a - # bytestring - results.append( (name.encode("utf-8"), - self._populate_row(keys, childnode) ) ) - return results - d.addCallback(_render) - d.addErrback(self._convert_error) - return d - - def openForReading(self, path): - d = self._get_node_and_metadata_for_path(path) - d.addCallback(lambda node_and_metadata: ReadFile(node_and_metadata[0])) - d.addErrback(self._convert_error) - return d - - def openForWriting(self, path): - path = [unicode(p) for p in path] - if not path: - raise ftp.PermissionDeniedError("cannot STOR to root directory") - childname = path[-1] - d = self._get_root(path) - def _got_root(root_and_path): - (root, path) = root_and_path - if not path: - raise ftp.PermissionDeniedError("cannot STOR to root directory") - return root.get_child_at_path(path[:-1]) - d.addCallback(_got_root) - def _got_parent(parent): - return WriteFile(parent, childname, self.convergence) - d.addCallback(_got_parent) - return d - -from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme - - -@implementer(portal.IRealm) -class Dispatcher(object): - def __init__(self, client): - self.client = client - - def requestAvatar(self, avatarID, mind, interface): - assert interface == ftp.IFTPShell - rootnode = self.client.create_node_from_uri(avatarID.rootcap) - convergence = self.client.convergence - s = Handler(self.client, rootnode, avatarID.username, convergence) - def logout(): pass - return (interface, s, None) - - -class FTPServer(service.MultiService): - def __init__(self, client, accountfile, accounturl, ftp_portstr): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) - service.MultiService.__init__(self) - - r = Dispatcher(client) - p = portal.Portal(r) - - if accountfile: - c = AccountFileChecker(self, accountfile) - p.registerChecker(c) - if accounturl: - c = AccountURLChecker(self, accounturl) - p.registerChecker(c) - if not accountfile and not accounturl: - # we could leave this anonymous, with just the /uri/CAP form - raise NeedRootcapLookupScheme("must provide some translation") - - f = ftp.FTPFactory(p) - # strports requires a native string. - ftp_portstr = ensure_str(ftp_portstr) - s = strports.service(ftp_portstr, f) - s.setServiceParent(self) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index b25ac0270..a86cde840 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1,6 +1,5 @@ import six import heapq, traceback, array, stat, struct -from types import NoneType from stat import S_IFREG, S_IFDIR from time import time, strftime, localtime @@ -267,7 +266,7 @@ def _attrs_to_metadata(attrs): def _direntry_for(filenode_or_parent, childname, filenode=None): - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) if childname is None: filenode_or_parent = filenode @@ -672,7 +671,7 @@ class GeneralSFTPFile(PrefixingLogMixin): self.log(".open(parent=%r, childname=%r, filenode=%r, metadata=%r)" % (parent, childname, filenode, metadata), level=OPERATIONAL) - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) precondition(filenode is None or IFileNode.providedBy(filenode), filenode=filenode) precondition(not self.closed, sftpfile=self) @@ -1194,7 +1193,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): request = "._sync_heisenfiles(%r, %r, ignore=%r)" % (userpath, direntry, ignore) self.log(request, level=OPERATIONAL) - _assert(isinstance(userpath, str) and isinstance(direntry, (str, NoneType)), + _assert(isinstance(userpath, str) and isinstance(direntry, (str, type(None))), userpath=userpath, direntry=direntry) files = [] @@ -1219,7 +1218,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): def _remove_heisenfile(self, userpath, parent, childname, file_to_remove): if noisy: self.log("._remove_heisenfile(%r, %r, %r, %r)" % (userpath, parent, childname, file_to_remove), level=NOISY) - _assert(isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)), + _assert(isinstance(userpath, str) and isinstance(childname, (unicode, type(None))), userpath=userpath, childname=childname) direntry = _direntry_for(parent, childname) @@ -1246,7 +1245,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata), level=NOISY) - _assert((isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)) and + _assert((isinstance(userpath, str) and isinstance(childname, (unicode, type(None))) and (metadata is None or 'no-write' in metadata)), userpath=userpath, childname=childname, metadata=metadata) @@ -1979,7 +1978,7 @@ class SFTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, sftp_portstr, pubkey_file, privkey_file): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) precondition(isinstance(pubkey_file, unicode), pubkey_file) precondition(isinstance(privkey_file, unicode), privkey_file) service.MultiService.__init__(self) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 68db478f3..6ed5571b9 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -19,7 +19,7 @@ from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI -@implementer(IImmutableFileNode, ICheckable) + class _ImmutableFileNodeBase(object): def get_write_uri(self): @@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object): return not self == other +@implementer(IImmutableFileNode, ICheckable) class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index d574b980d..2d2c5c1f5 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object): @implementer(interfaces.RICHKUploadHelper) -class CHKUploadHelper(Referenceable, upload.CHKUploader): +class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warner/foolscap#78 """I am the helper-server -side counterpart to AssistedUploader. I handle peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. @@ -499,10 +499,13 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561 + def set_upload_status(self, upload_status): + raise NotImplementedError @implementer(interfaces.RIHelper, interfaces.IStatsProducer) -class Helper(Referenceable): +class Helper(Referenceable): # type: ignore # warner/foolscap#78 """ :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which have been started but not finished, a mapping from storage index to the diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 6dae825ac..46e01184f 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -13,19 +13,30 @@ if PY2: from past.builtins import long, unicode from six import ensure_str +try: + from typing import List +except ImportError: + pass + import os, time, weakref, itertools + +import attr + from zope.interface import implementer from twisted.python import failure from twisted.internet import defer from twisted.application import service -from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually +from foolscap.api import Referenceable, Copyable, RemoteCopy from allmydata.crypto import aes from allmydata.util.hashutil import file_renewal_secret_hash, \ file_cancel_secret_hash, bucket_renewal_secret_hash, \ bucket_cancel_secret_hash, plaintext_hasher, \ storage_index_hash, plaintext_segment_hasher, convergence_hasher -from allmydata.util.deferredutil import timeout_call +from allmydata.util.deferredutil import ( + timeout_call, + until, +) from allmydata import hashtree, uri from allmydata.storage.server import si_b2a from allmydata.immutable import encode @@ -386,6 +397,9 @@ class PeerSelector(object): ) return self.happiness_mappings + def add_peers(self, peerids=None): + raise NotImplementedError + class _QueryStatistics(object): @@ -897,13 +911,45 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): raise UploadUnhappinessError(msg) +@attr.s +class _Accum(object): + """ + Accumulate up to some known amount of ciphertext. + + :ivar remaining: The number of bytes still expected. + :ivar ciphertext: The bytes accumulated so far. + """ + remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int + ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes] + + def extend(self, + size, # type: int + ciphertext, # type: List[bytes] + ): + """ + Accumulate some more ciphertext. + + :param size: The amount of data the new ciphertext represents towards + the goal. This may be more than the actual size of the given + ciphertext if the source has run out of data. + + :param ciphertext: The new ciphertext to accumulate. + """ + self.remaining -= size + self.ciphertext.extend(ciphertext) + + @implementer(IEncryptedUploadable) class EncryptAnUploadable(object): """This is a wrapper that takes an IUploadable and provides IEncryptedUploadable.""" CHUNKSIZE = 50*1024 - def __init__(self, original, log_parent=None, progress=None): + def __init__(self, original, log_parent=None, progress=None, chunk_size=None): + """ + :param chunk_size: The number of bytes to read from the uploadable at a + time, or None for some default. + """ precondition(original.default_params_set, "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,)) self.original = IUploadable(original) @@ -917,6 +963,8 @@ class EncryptAnUploadable(object): self._ciphertext_bytes_read = 0 self._status = None self._progress = progress + if chunk_size is not None: + self.CHUNKSIZE = chunk_size def set_upload_status(self, upload_status): self._status = IUploadStatus(upload_status) @@ -1023,47 +1071,53 @@ class EncryptAnUploadable(object): # and size d.addCallback(lambda ignored: self.get_size()) d.addCallback(lambda ignored: self._get_encryptor()) - # then fetch and encrypt the plaintext. The unusual structure here - # (passing a Deferred *into* a function) is needed to avoid - # overflowing the stack: Deferreds don't optimize out tail recursion. - # We also pass in a list, to which _read_encrypted will append - # ciphertext. - ciphertext = [] - d2 = defer.Deferred() - d.addCallback(lambda ignored: - self._read_encrypted(length, ciphertext, hash_only, d2)) - d.addCallback(lambda ignored: d2) + + accum = _Accum(length) + + def action(): + """ + Read some bytes into the accumulator. + """ + return self._read_encrypted(accum, hash_only) + + def condition(): + """ + Check to see if the accumulator has all the data. + """ + return accum.remaining == 0 + + d.addCallback(lambda ignored: until(action, condition)) + d.addCallback(lambda ignored: accum.ciphertext) return d - def _read_encrypted(self, remaining, ciphertext, hash_only, fire_when_done): - if not remaining: - fire_when_done.callback(ciphertext) - return None + def _read_encrypted(self, + ciphertext_accum, # type: _Accum + hash_only, # type: bool + ): + # type: (...) -> defer.Deferred + """ + Read the next chunk of plaintext, encrypt it, and extend the accumulator + with the resulting ciphertext. + """ # tolerate large length= values without consuming a lot of RAM by # reading just a chunk (say 50kB) at a time. This only really matters # when hash_only==True (i.e. resuming an interrupted upload), since # that's the case where we will be skipping over a lot of data. - size = min(remaining, self.CHUNKSIZE) - remaining = remaining - size + size = min(ciphertext_accum.remaining, self.CHUNKSIZE) + # read a chunk of plaintext.. d = defer.maybeDeferred(self.original.read, size) - # N.B.: if read() is synchronous, then since everything else is - # actually synchronous too, we'd blow the stack unless we stall for a - # tick. Once you accept a Deferred from IUploadable.read(), you must - # be prepared to have it fire immediately too. - d.addCallback(fireEventually) def _good(plaintext): # and encrypt it.. # o/' over the fields we go, hashing all the way, sHA! sHA! sHA! o/' ct = self._hash_and_encrypt_plaintext(plaintext, hash_only) - ciphertext.extend(ct) - self._read_encrypted(remaining, ciphertext, hash_only, - fire_when_done) - def _err(why): - fire_when_done.errback(why) + # Intentionally tell the accumulator about the expected size, not + # the actual size. If we run out of data we still want remaining + # to drop otherwise it will never reach 0 and the loop will never + # end. + ciphertext_accum.extend(size, ct) d.addCallback(_good) - d.addErrback(_err) - return None + return d def _hash_and_encrypt_plaintext(self, data, hash_only): assert isinstance(data, (tuple, list)), type(data) @@ -1424,7 +1478,7 @@ class LiteralUploader(object): return self._status @implementer(RIEncryptedUploadable) -class RemoteEncryptedUploadable(Referenceable): +class RemoteEncryptedUploadable(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, encrypted_uploadable, upload_status): self._eu = IEncryptedUploadable(encrypted_uploadable) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 6d0938dd5..0dd5ddc83 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -681,7 +681,7 @@ class IURI(Interface): passing into init_from_string.""" -class IVerifierURI(Interface, IURI): +class IVerifierURI(IURI): def init_from_string(uri): """Accept a string (as created by my to_string() method) and populate this instance with its data. I am not normally called directly, @@ -748,7 +748,7 @@ class IProgress(Interface): "Current amount of progress (in percentage)" ) - def set_progress(self, value): + def set_progress(value): """ Sets the current amount of progress. @@ -756,7 +756,7 @@ class IProgress(Interface): set_progress_total. """ - def set_progress_total(self, value): + def set_progress_total(value): """ Sets the total amount of expected progress @@ -859,12 +859,6 @@ class IPeerSelector(Interface): peer selection begins. """ - def confirm_share_allocation(peerid, shnum): - """ - Confirm that an allocated peer=>share pairing has been - successfully established. - """ - def add_peers(peerids=set): """ Update my internal state to include the peers in peerids as @@ -1824,11 +1818,6 @@ class IEncoder(Interface): willing to receive data. """ - def set_size(size): - """Specify the number of bytes that will be encoded. This must be - peformed before get_serialized_params() can be called. - """ - def set_encrypted_uploadable(u): """Provide a source of encrypted upload data. 'u' must implement IEncryptedUploadable. diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 72d68acec..07f8a5f7a 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -178,9 +178,9 @@ class IntroducerClient(service.Service, Referenceable): kwargs["facility"] = "tahoe.introducer.client" return log.msg(*args, **kwargs) - def subscribe_to(self, service_name, cb, *args, **kwargs): + def subscribe_to(self, service_name, callback, *args, **kwargs): obs = self._local_subscribers.setdefault(service_name, ObserverList()) - obs.subscribe(lambda key_s, ann: cb(key_s, ann, *args, **kwargs)) + obs.subscribe(lambda key_s, ann: callback(key_s, ann, *args, **kwargs)) self._maybe_subscribe() for index,(ann,key_s,when) in list(self._inbound_announcements.items()): precondition(isinstance(key_s, bytes), key_s) diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 9f08f1943..24fd3945f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -73,7 +73,7 @@ class IIntroducerClient(Interface): publish their services to the rest of the world, and I help them learn about services available on other nodes.""" - def publish(service_name, ann, signing_key=None): + def publish(service_name, ann, signing_key): """Publish the given announcement dictionary (which must be JSON-serializable), plus some additional keys, to the world. @@ -83,8 +83,7 @@ class IIntroducerClient(Interface): the signing_key, if present, otherwise it is derived from the 'anonymous-storage-FURL' key. - If signing_key= is set to an instance of SigningKey, it will be - used to sign the announcement.""" + signing_key (a SigningKey) will be used to sign the announcement.""" def subscribe_to(service_name, callback, *args, **kwargs): """Call this if you will eventually want to use services with the diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 237c30315..339c5a0ac 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -15,6 +15,12 @@ from past.builtins import long from six import ensure_text import time, os.path, textwrap + +try: + from typing import Any, Dict, Union +except ImportError: + pass + from zope.interface import implementer from twisted.application import service from twisted.internet import defer @@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable): name = "introducer" # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 - VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, + # TODO: reconcile bytes/str for keys + VERSION = { + #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } + } # type: Dict[Union[bytes, str], Any] def __init__(self): service.MultiService.__init__(self) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 5afc84dec..39e8b76be 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap): + def upload(self, new_contents, servermap, progress=None): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of @@ -951,7 +951,7 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): + def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. @@ -1205,3 +1205,7 @@ class MutableFileVersion(object): self._servermap, mode=mode) return u.update() + + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562 + def get_servermap(self): + raise NotImplementedError diff --git a/src/allmydata/node.py b/src/allmydata/node.py index e08c07508..2f340f860 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -23,6 +23,11 @@ from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn +try: + from typing import Union +except ImportError: + pass + import attr # On Python 2 this will be the backported package. @@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files): raise e +def ensure_text_and_abspath_expanduser_unicode(basedir): + # type: (Union[bytes, str]) -> str + return abspath_expanduser_unicode(ensure_text(basedir)) + + @attr.s class _Config(object): """ @@ -300,8 +310,8 @@ class _Config(object): config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser)) portnum_fname = attr.ib() _basedir = attr.ib( - converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)), - ) + converter=ensure_text_and_abspath_expanduser_unicode, + ) # type: str config_path = attr.ib( validator=attr.validators.optional( attr.validators.instance_of(FilePath), @@ -927,7 +937,6 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e472ffd8c..50dde9e43 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import BaseOptions @@ -79,8 +84,8 @@ def do_admin(options): subCommands = [ - ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] + ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), + ] # type: SubCommands dispatch = { "admin": do_admin, diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 379e1d212..e4cd8aa22 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,12 @@ from __future__ import print_function import os.path, re, fnmatch + +try: + from allmydata.scripts.types_ import SubCommands, Parameters +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions @@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] + ] # type: Parameters def postOptions(self): self["quiet"] = self.parent["quiet"] @@ -455,25 +461,25 @@ class DeepCheckOptions(FileStoreOptions): Optionally repair any problems found.""" subCommands = [ - ["mkdir", None, MakeDirectoryOptions, "Create a new directory."], - ["add-alias", None, AddAliasOptions, "Add a new alias cap."], - ["create-alias", None, CreateAliasOptions, "Create a new alias cap."], - ["list-aliases", None, ListAliasesOptions, "List all alias caps."], - ["ls", None, ListOptions, "List a directory."], - ["get", None, GetOptions, "Retrieve a file from the grid."], - ["put", None, PutOptions, "Upload a file into the grid."], - ["cp", None, CpOptions, "Copy one or more files or directories."], - ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["mv", None, MvOptions, "Move a file within the grid."], - ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], - ["backup", None, BackupOptions, "Make target dir look like local dir."], - ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."], - ["manifest", None, ManifestOptions, "List all files/directories in a subtree."], - ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."], - ["check", None, CheckOptions, "Check a single file or directory."], - ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."], - ["status", None, TahoeStatusCommand, "Various status information."], - ] + ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), + ("add-alias", None, AddAliasOptions, "Add a new alias cap."), + ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), + ("list-aliases", None, ListAliasesOptions, "List all alias caps."), + ("ls", None, ListOptions, "List a directory."), + ("get", None, GetOptions, "Retrieve a file from the grid."), + ("put", None, PutOptions, "Upload a file into the grid."), + ("cp", None, CpOptions, "Copy one or more files or directories."), + ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."), + ("mv", None, MvOptions, "Move a file within the grid."), + ("ln", None, LnOptions, "Make an additional link to an existing file or directory."), + ("backup", None, BackupOptions, "Make target dir look like local dir."), + ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."), + ("manifest", None, ManifestOptions, "List all files/directories in a subtree."), + ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."), + ("check", None, CheckOptions, "Check a single file or directory."), + ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), + ("status", None, TahoeStatusCommand, "Various status information."), + ] # type: SubCommands def mkdir(options): from allmydata.scripts import tahoe_mkdir diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 106dad3f2..d73344274 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,6 +4,12 @@ import os, sys, urllib, textwrap import codecs from os.path import join +try: + from typing import Optional + from .types_ import Parameters +except ImportError: + pass + from yaml import ( safe_dump, ) @@ -41,8 +47,8 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None - description_unwrapped = None + description = None # type: Optional[str] + description_unwrapped = None # type: Optional[str] def __str__(self): width = int(os.environ.get('COLUMNS', '80')) @@ -65,7 +71,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] + ] # type: Parameters def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -102,7 +108,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] + ] # type: Parameters # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index ac17cf445..0f507f518 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -3,6 +3,11 @@ from __future__ import print_function import os import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.internet import reactor, defer from twisted.python.usage import UsageError from twisted.python.filepath import ( @@ -492,10 +497,10 @@ def create_introducer(config): subCommands = [ - ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."], - ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."], - ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."], -] + ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), + ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), + ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), +] # type: SubCommands dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 451b1d661..550c37fde 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from future.utils import bchr # do not import any allmydata modules at this level. Do that from inside @@ -1053,8 +1058,8 @@ def do_debug(options): subCommands = [ - ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."], - ] + ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), + ] # type: SubCommands dispatch = { "debug": do_debug, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 1f993fda1..9a632a57d 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,11 @@ import os, sys from six.moves import StringIO import six +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, task, threads @@ -40,8 +45,8 @@ _control_node_dispatch = { } process_control_commands = [ - ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], -] + ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), +] # type: SubCommands class Options(usage.Options): @@ -98,7 +103,7 @@ class Options(usage.Options): create_dispatch = {} for module in (create_node,): - create_dispatch.update(module.dispatch) + create_dispatch.update(module.dispatch) # type: ignore def parse_options(argv, config=None): if not config: diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index dbc84d0ea..884536ec2 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -2,6 +2,11 @@ from __future__ import print_function import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, reactor @@ -103,7 +108,7 @@ def invite(options): subCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] +] # type: SubCommands dispatch = { "invite": invite, diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py new file mode 100644 index 000000000..3937cb803 --- /dev/null +++ b/src/allmydata/scripts/types_.py @@ -0,0 +1,12 @@ +from typing import List, Tuple, Type, Sequence, Any +from allmydata.scripts.common import BaseOptions + + +# Historically, subcommands were implemented as lists, but due to a +# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170), +# a Tuple is required. +SubCommand = Tuple[str, None, Type[BaseOptions], str] + +SubCommands = List[SubCommand] + +Parameters = List[Sequence[Any]] diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 18b22c30a..91205a93c 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -1,11 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from __future__ import unicode_literals -import time - -# Python 2 compatibility from future.utils import PY2 if PY2: - from future.builtins import str # noqa: F401 + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import time from twisted.application import service from twisted.application.internet import TimerService @@ -18,7 +23,7 @@ from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 + POLL_INTERVAL = 60 # type: float def __init__(self): service.MultiService.__init__(self) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 24042c38b..f13f7cb99 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,7 +19,7 @@ import os, time, struct try: import cPickle as pickle except ImportError: - import pickle + import pickle # type: ignore from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 778c0ddf8..4b60d79f1 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -202,7 +202,7 @@ class ShareFile(object): @implementer(RIBucketWriter) -class BucketWriter(Referenceable): +class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): self.ss = ss @@ -301,7 +301,7 @@ class BucketWriter(Referenceable): @implementer(RIBucketReader) -class BucketReader(Referenceable): +class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8a8138f26..5f2ef3ac2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( + def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 4058ddf77..21576ea3a 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse +try: + from typing import Dict +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -49,13 +54,13 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} + last_stats = {} # type: Dict[str, float] while True: - stats = {} + stats = {} # type: Dict[str, float] for sf in statsfiles: for line in open(sf, "r").readlines(): - name, value = line.split(":") - value = int(value.strip()) + name, str_value = line.split(":") + value = int(str_value.strip()) if name not in stats: stats[name] = 0 stats[name] += float(value) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 6ec90eeae..268d77451 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -508,13 +508,13 @@ if __name__ == '__main__': mode = "upload" if len(sys.argv) > 1: mode = sys.argv[1] - if sys.maxint == 2147483647: + if sys.maxsize == 2147483647: bits = "32" - elif sys.maxint == 9223372036854775807: + elif sys.maxsize == 9223372036854775807: bits = "64" else: bits = "?" - print("%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)) + print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) # put the logfile and stats.out in _test_memory/ . These stick around. # put the nodes and other files in _test_memory/test/ . These are # removed each time we run. diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index f1dbf651d..fde92fb59 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -406,7 +406,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode(object): +class FakeCHKFileNode(object): # type: ignore # incomplete implementation """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -544,7 +544,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode(object): +class FakeMutableFileNode(object): # type: ignore # incomplete implementation """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index e1f04b864..cbea0dfcd 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -68,7 +68,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) +@implementer(IRemoteReference) # type: ignore # warner/foolscap#79 class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local @@ -213,9 +213,12 @@ class NoNetworkServer(object): return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version + def start_connecting(self, trigger_cb): + raise NotImplementedError + @implementer(IStorageBroker) -class NoNetworkStorageBroker(object): +class NoNetworkStorageBroker(object): # type: ignore # missing many methods def get_servers_for_psi(self, peer_selection_index): def _permuted(server): seed = server.get_permutation_seed() @@ -259,7 +262,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): +class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 """ Overrides all _Client networking functionality to do nothing. """ diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 4a1f84531..17ec89078 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -47,8 +47,9 @@ class RIDummy(RemoteInterface): """ - -@implementer(IFoolscapStoragePlugin) +# type ignored due to missing stubs for Twisted +# https://twistedmatrix.com/trac/ticket/9717 +@implementer(IFoolscapStoragePlugin) # type: ignore @attr.s class DummyStorage(object): name = attr.ib() @@ -107,7 +108,7 @@ class GetCounter(Resource, object): @implementer(RIDummy) @attr.s(frozen=True) -class DummyStorageServer(object): +class DummyStorageServer(object): # type: ignore # warner/foolscap#78 get_anonymous_storage_server = attr.ib() def remote_just_some_method(self): @@ -116,7 +117,7 @@ class DummyStorageServer(object): @implementer(IStorageServer) @attr.s -class DummyStorageClient(object): +class DummyStorageClient(object): # type: ignore # incomplete implementation get_rref = attr.ib() configuration = attr.ib() announcement = attr.ib() diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index a7042468a..f56ecd089 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): +class FakeServer(object): # type: ignore # incomplete implementation def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeCheckResults(object): +class FakeCheckResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, repair_attempted=False, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 342fe4af1..63a5ceaaa 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -51,7 +51,6 @@ from allmydata.nodemaker import ( NodeMaker, ) from allmydata.node import OldConfigError, UnescapedHashError, create_node_dir -from allmydata.frontends.auth import NeedRootcapLookupScheme from allmydata import client from allmydata.storage_client import ( StorageClientConfig, @@ -424,88 +423,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir) self.failUnlessReallyEqual(w.staticdir, expected) - # TODO: also test config options for SFTP. - - @defer.inlineCallbacks - def test_ftp_create(self): - """ - configuration for sftpd results in it being started - """ - root = FilePath(self.mktemp()) - root.makedirs() - accounts = root.child(b"sftp-accounts") - accounts.touch() - - data = FilePath(__file__).sibling(b"data") - privkey = data.child(b"openssh-rsa-2048.txt") - pubkey = data.child(b"openssh-rsa-2048.pub.txt") - - basedir = u"client.Basic.test_ftp_create" - create_node_dir(basedir, "testing") - with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: - f.write(( - '[sftpd]\n' - 'enabled = true\n' - 'accounts.file = {}\n' - 'host_pubkey_file = {}\n' - 'host_privkey_file = {}\n' - ).format(accounts.path, pubkey.path, privkey.path)) - - client_node = yield client.create_client( - basedir, - ) - sftp = client_node.getServiceNamed("frontend:sftp") - self.assertIs(sftp.parent, client_node) - - - @defer.inlineCallbacks - def test_ftp_auth_keyfile(self): - """ - ftpd accounts.file is parsed properly - """ - basedir = u"client.Basic.test_ftp_auth_keyfile" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n" - "accounts.file = private/accounts\n")) - os.mkdir(os.path.join(basedir, "private")) - fileutil.write(os.path.join(basedir, "private", "accounts"), "\n") - c = yield client.create_client(basedir) # just make sure it can be instantiated - del c - - @defer.inlineCallbacks - def test_ftp_auth_url(self): - """ - ftpd accounts.url is parsed properly - """ - basedir = u"client.Basic.test_ftp_auth_url" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n" - "accounts.url = http://0.0.0.0/\n")) - c = yield client.create_client(basedir) # just make sure it can be instantiated - del c - - @defer.inlineCallbacks - def test_ftp_auth_no_accountfile_or_url(self): - """ - ftpd requires some way to look up accounts - """ - basedir = u"client.Basic.test_ftp_auth_no_accountfile_or_url" - os.mkdir(basedir) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), - (BASECONFIG + - "[ftpd]\n" - "enabled = true\n" - "port = tcp:0:interface=127.0.0.1\n")) - with self.assertRaises(NeedRootcapLookupScheme): - yield client.create_client(basedir) + # TODO: also test config options for SFTP. See Git history for deleted FTP + # tests that could be used as basis for these tests. @defer.inlineCallbacks def _storage_dir_test(self, basedir, storage_path, expected_path): diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 6ebc93556..2a155089f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -74,3 +74,58 @@ class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin d = defer.succeed(None) d.addBoth(self.wait_for_delayed_calls) return d + + +class UntilTests(unittest.TestCase): + """ + Tests for ``deferredutil.until``. + """ + def test_exception(self): + """ + If the action raises an exception, the ``Deferred`` returned by ``until`` + fires with a ``Failure``. + """ + self.assertFailure( + deferredutil.until(lambda: 1/0, lambda: True), + ZeroDivisionError, + ) + + def test_stops_on_condition(self): + """ + The action is called repeatedly until ``condition`` returns ``True``. + """ + calls = [] + def action(): + calls.append(None) + + def condition(): + return len(calls) == 3 + + self.assertIs( + self.successResultOf( + deferredutil.until(action, condition), + ), + None, + ) + self.assertEqual(3, len(calls)) + + def test_waits_for_deferred(self): + """ + If the action returns a ``Deferred`` then it is called again when the + ``Deferred`` fires. + """ + counter = [0] + r1 = defer.Deferred() + r2 = defer.Deferred() + results = [r1, r2] + def action(): + counter[0] += 1 + return results.pop(0) + + def condition(): + return False + + deferredutil.until(action, condition) + self.assertEqual([1], counter) + r1.callback(None) + self.assertEqual([2], counter) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 1c265492b..8e5e59b46 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1561,7 +1561,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): +class FakeMutableFile(object): # type: ignore # incomplete implementation counter = 0 def __init__(self, initial_contents=b""): data = self._get_initial_contents(initial_contents) @@ -1622,7 +1622,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents=b"", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): +class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, diff --git a/src/allmydata/test/test_ftp.py b/src/allmydata/test/test_ftp.py deleted file mode 100644 index 4eddef440..000000000 --- a/src/allmydata/test/test_ftp.py +++ /dev/null @@ -1,106 +0,0 @@ - -from twisted.trial import unittest - -from allmydata.frontends import ftpd -from allmydata.immutable import upload -from allmydata.mutable import publish -from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_util import ReallyEqualMixin - -class Handler(GridTestMixin, ReallyEqualMixin, unittest.TestCase): - """ - This is a no-network unit test of ftpd.Handler and the abstractions - it uses. - """ - - FALL_OF_BERLIN_WALL = 626644800 - TURN_OF_MILLENIUM = 946684800 - - def _set_up(self, basedir, num_clients=1, num_servers=10): - self.basedir = "ftp/" + basedir - self.set_up_grid(num_clients=num_clients, num_servers=num_servers, - oneshare=True) - - self.client = self.g.clients[0] - self.username = "alice" - self.convergence = "" - - d = self.client.create_dirnode() - def _created_root(node): - self.root = node - self.root_uri = node.get_uri() - self.handler = ftpd.Handler(self.client, self.root, self.username, - self.convergence) - d.addCallback(_created_root) - return d - - def _set_metadata(self, name, metadata): - """Set metadata for `name', avoiding MetadataSetter's timestamp reset - behavior.""" - def _modifier(old_contents, servermap, first_time): - children = self.root._unpack_contents(old_contents) - children[name] = (children[name][0], metadata) - return self.root._pack_contents(children) - - return self.root._node.modify(_modifier) - - def _set_up_tree(self): - # add immutable file at root - immutable = upload.Data("immutable file contents", None) - d = self.root.add_file(u"immutable", immutable) - - # `mtime' and `linkmotime' both set - md_both = {'mtime': self.FALL_OF_BERLIN_WALL, - 'tahoe': {'linkmotime': self.TURN_OF_MILLENIUM}} - d.addCallback(lambda _: self._set_metadata(u"immutable", md_both)) - - # add link to root from root - d.addCallback(lambda _: self.root.set_node(u"loop", self.root)) - - # `mtime' set, but no `linkmotime' - md_just_mtime = {'mtime': self.FALL_OF_BERLIN_WALL, 'tahoe': {}} - d.addCallback(lambda _: self._set_metadata(u"loop", md_just_mtime)) - - # add mutable file at root - mutable = publish.MutableData("mutable file contents") - d.addCallback(lambda _: self.client.create_mutable_file(mutable)) - d.addCallback(lambda node: self.root.set_node(u"mutable", node)) - - # neither `mtime' nor `linkmotime' set - d.addCallback(lambda _: self._set_metadata(u"mutable", {})) - - return d - - def _compareDirLists(self, actual, expected): - actual_list = sorted(actual) - expected_list = sorted(expected) - - self.failUnlessReallyEqual(len(actual_list), len(expected_list), - "%r is wrong length, expecting %r" % ( - actual_list, expected_list)) - for (a, b) in zip(actual_list, expected_list): - (name, meta) = a - (expected_name, expected_meta) = b - self.failUnlessReallyEqual(name, expected_name) - self.failUnlessReallyEqual(meta, expected_meta) - - def test_list(self): - keys = ("size", "directory", "permissions", "hardlinks", "modified", - "owner", "group", "unexpected") - d = self._set_up("list") - - d.addCallback(lambda _: self._set_up_tree()) - d.addCallback(lambda _: self.handler.list("", keys=keys)) - - expected_root = [ - ('loop', - [0, True, ftpd.IntishPermissions(0o600), 1, self.FALL_OF_BERLIN_WALL, 'alice', 'alice', '??']), - ('immutable', - [23, False, ftpd.IntishPermissions(0o600), 1, self.TURN_OF_MILLENIUM, 'alice', 'alice', '??']), - ('mutable', - # timestamp should be 0 if no timestamp metadata is present - [0, False, ftpd.IntishPermissions(0o600), 1, 0, 'alice', 'alice', '??'])] - - d.addCallback(lambda root: self._compareDirLists(root, expected_root)) - - return d diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 65c07135a..3faffbe0d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -19,6 +19,12 @@ from functools import ( ) import attr +try: + from typing import List + from allmydata.introducer.client import IntroducerClient +except ImportError: + pass + from twisted.internet import defer from twisted.trial import unittest from twisted.application import service @@ -125,7 +131,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] + introducer_clients = [] # type: List[IntroducerClient] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..e44fd5743 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -564,7 +564,7 @@ class TestMissingPorts(unittest.TestCase): config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( + test_listen_on_zero_with_host.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" ) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 84484f1cf..5c6a654c1 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -16,6 +16,7 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.test.check_load", "allmydata.windows.registry", + "allmydata.scripts.types_", } diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 80242f8a2..c1f0e83d6 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -44,7 +44,7 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = native_str( + test_finished_porting.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", ) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index b6f1fbc8a..1ff0363e8 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -9,18 +9,15 @@ from twisted.python.failure import Failure from twisted.internet.error import ProcessDone, ProcessTerminated from allmydata.util import deferredutil -conch_interfaces = None -sftp = None -sftpd = None - try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd except ImportError as e: + conch_interfaces = sftp = sftpd = None # type: ignore conch_unavailable_reason = e else: - conch_unavailable_reason = None + conch_unavailable_reason = None # type: ignore from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError diff --git a/src/allmydata/test/test_stats.py b/src/allmydata/test/test_stats.py index 3ee495927..e56f9d444 100644 --- a/src/allmydata/test/test_stats.py +++ b/src/allmydata/test/test_stats.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.application import service diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 3a21dfd9e..8500d6bff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -105,7 +105,8 @@ from allmydata.interfaces import ( SOME_FURL = "pb://abcde@nowhere/fake" -class NativeStorageServerWithVersion(NativeStorageServer): + +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall @@ -569,7 +570,7 @@ class SpyEndpoint(object): return d -@implementer(IConnectionHintHandler) +@implementer(IConnectionHintHandler) # type: ignore # warner/foolscap#78 @attr.s class SpyHandler(object): """ diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 94d7575c3..07ede2074 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -14,6 +14,17 @@ if PY2: import os, shutil from io import BytesIO +from base64 import ( + b64encode, +) + +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + just, + integers, +) from twisted.trial import unittest from twisted.python.failure import Failure @@ -2029,6 +2040,91 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, f.close() return None + +class EncryptAnUploadableTests(unittest.TestCase): + """ + Tests for ``EncryptAnUploadable``. + """ + def test_same_length(self): + """ + ``EncryptAnUploadable.read_encrypted`` returns ciphertext of the same + length as the underlying plaintext. + """ + plaintext = b"hello world" + uploadable = upload.FileHandle(BytesIO(plaintext), None) + uploadable.set_default_encoding_parameters({ + # These values shouldn't matter. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + ciphertext = b"".join(self.successResultOf(encrypter.read_encrypted(1024, False))) + self.assertEqual(len(ciphertext), len(plaintext)) + + @given(just(b"hello world"), integers(min_value=0, max_value=len(b"hello world"))) + def test_known_result(self, plaintext, split_at): + """ + ``EncryptAnUploadable.read_encrypted`` returns a known-correct ciphertext + string for certain inputs. The ciphertext is independent of the read + sizes. + """ + convergence = b"\x42" * 16 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + # The convergence key is a function of k, n, and max_segment_size + # (among other things). The value for happy doesn't matter + # though. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + def read(n): + return b"".join(self.successResultOf(encrypter.read_encrypted(n, False))) + + # Read the string in one or two pieces to make sure underlying state + # is maintained properly. + first = read(split_at) + second = read(len(plaintext) - split_at) + third = read(1) + ciphertext = first + second + third + + self.assertEqual( + b"Jd2LHCRXozwrEJc=", + b64encode(ciphertext), + ) + + def test_large_read(self): + """ + ``EncryptAnUploadable.read_encrypted`` succeeds even when the requested + data length is much larger than the chunk size. + """ + convergence = b"\x42" * 16 + # 4kB of plaintext + plaintext = b"\xde\xad\xbe\xef" * 1024 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + # Make the chunk size very small so we don't have to operate on a huge + # amount of data to exercise the relevant codepath. + encrypter = upload.EncryptAnUploadable(uploadable, chunk_size=1) + d = encrypter.read_encrypted(len(plaintext), False) + ciphertext = self.successResultOf(d) + self.assertEqual( + list(map(len, ciphertext)), + # Chunk size was specified as 1 above so we will get the whole + # plaintext in one byte chunks. + [1] * len(plaintext), + ) + + # TODO: # upload with exactly 75 servers (shares_of_happiness) # have a download fail diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py deleted file mode 100644 index e666a4902..000000000 --- a/src/allmydata/test/test_websocket_logs.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from twisted.trial import unittest -from twisted.internet.defer import inlineCallbacks - -from eliot import log_call - -from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper - -from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol - - -class TestStreamingLogs(unittest.TestCase): - """ - Test websocket streaming of logs - """ - - def setUp(self): - self.reactor = MemoryReactorClockResolver() - self.pumper = create_pumper() - self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) - return self.pumper.start() - - def tearDown(self): - return self.pumper.stop() - - @inlineCallbacks - def test_one_log(self): - """ - write a single Eliot log and see it streamed via websocket - """ - - proto = yield self.agent.open( - transport_config=u"ws://localhost:1234/ws", - options={}, - ) - - messages = [] - def got_message(msg, is_binary=False): - messages.append(json.loads(msg)) - proto.on("message", got_message) - - @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): - pass - - do_a_thing() - - proto.transport.loseConnection() - yield proto.is_closed - - self.assertEqual(len(messages), 2) - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 929fba507..08d95bda9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import json from os.path import join @@ -213,7 +225,7 @@ class IntroducerRootTests(unittest.TestCase): resource = IntroducerRoot(introducer_node) response = json.loads( self.successResultOf( - render(resource, {"t": [b"json"]}), + render(resource, {b"t": [b"json"]}), ), ) self.assertEqual( diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 4895ed6f0..5d697f910 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.logs``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,19 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import json + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from eliot import log_call + +from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper + from testtools.matchers import ( Equals, ) @@ -37,6 +52,7 @@ from ..common import ( from ...web.logs import ( create_log_resources, + TokenAuthenticatedWebSocketServerProtocol, ) class StreamingEliotLogsTests(SyncTestCase): @@ -57,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase): self.client.get(b"http:///v1"), succeeded(has_response_code(Equals(OK))), ) + + +class TestStreamingLogs(unittest.TestCase): + """ + Test websocket streaming of logs + """ + + def setUp(self): + self.reactor = MemoryReactorClockResolver() + self.pumper = create_pumper() + self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) + return self.pumper.start() + + def tearDown(self): + return self.pumper.stop() + + @inlineCallbacks + def test_one_log(self): + """ + write a single Eliot log and see it streamed via websocket + """ + + proto = yield self.agent.open( + transport_config=u"ws://localhost:1234/ws", + options={}, + ) + + messages = [] + def got_message(msg, is_binary=False): + messages.append(json.loads(msg)) + proto.on("message", got_message) + + @log_call(action_type=u"test:cli:some-exciting-action") + def do_a_thing(): + pass + + do_a_thing() + + proto.transport.loseConnection() + yield proto.is_closed + + self.assertEqual(len(messages), 2) + self.assertEqual("started", messages[0]["action_status"]) + self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 27ddbcf78..b426b4d93 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.private``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from testtools.matchers import ( Equals, ) @@ -56,6 +62,7 @@ class PrivacyTests(SyncTestCase): return super(PrivacyTests, self).setUp() def _authorization(self, scheme, value): + value = str(value, "utf-8") return Headers({ u"authorization": [u"{} {}".format(scheme, value)], }) @@ -90,7 +97,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, u"foo bar"), + headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) @@ -103,7 +110,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, self.token), + headers=self._authorization(str(SCHEME, "utf-8"), self.token), ), # It's a made up URL so we don't get a 200, either, but a 404. succeeded(has_response_code(Equals(NOT_FOUND))), diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..ca3cc695d 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,6 +1,18 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time -from urllib import ( +from urllib.parse import ( quote, ) @@ -77,7 +89,7 @@ class RenderSlashUri(unittest.TestCase): ) self.assertEqual( response_body, - "Invalid capability", + b"Invalid capability", ) @@ -92,7 +104,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) class FakeClient(_Client): @@ -103,7 +115,7 @@ class RenderServiceRow(unittest.TestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server("test-srv", srv) + self.storage_broker.test_add_server(b"test-srv", srv) root = RootElement(FakeClient(), time.time) req = DummyRequest(b"") diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e975464d3..ce9a40389 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): +class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): +class FakeClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index e680acd04..12a04a6eb 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.webish``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from uuid import ( uuid4, @@ -96,7 +106,7 @@ class TahoeLAFSRequestTests(SyncTestCase): ]) self._fields_test( b"POST", - {b"content-type": b"multipart/form-data; boundary={}".format(boundary)}, + {b"content-type": b"multipart/form-data; boundary=" + bytes(boundary, 'ascii')}, form_data.encode("ascii"), AfterPreprocessing( lambda fs: { @@ -105,8 +115,8 @@ class TahoeLAFSRequestTests(SyncTestCase): in fs.keys() }, Equals({ - b"foo": b"bar", - b"baz": b"some file contents", + "foo": "bar", + "baz": b"some file contents", }), ), ) diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index f79c88415..060696293 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -1,3 +1,13 @@ +"""Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 2c367cafe..51671b0ac 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -22,6 +22,11 @@ from past.builtins import unicode, long import re +try: + from typing import Type +except ImportError: + pass + from zope.interface import implementer from twisted.python.components import registerAdapter @@ -489,7 +494,7 @@ class MDMFVerifierURI(_BaseURI): return self -@implementer(IURI, IDirnodeURI) +@implementer(IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri @@ -536,7 +541,7 @@ class _DirectoryBaseURI(_BaseURI): return self._filenode_uri.get_storage_index() -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2:' @@ -555,7 +560,7 @@ class DirectoryURI(_DirectoryBaseURI): return ReadonlyDirectoryURI(self._filenode_uri.get_readonly()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-RO:' @@ -574,6 +579,7 @@ class ReadonlyDirectoryURI(_DirectoryBaseURI): return self +@implementer(IURI, IDirnodeURI) class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): def __init__(self, filenode_uri=None): if filenode_uri: @@ -611,7 +617,7 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): return None -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF:' @@ -633,7 +639,7 @@ class MDMFDirectoryURI(_DirectoryBaseURI): return MDMFDirectoryURIVerifier(self._filenode_uri.get_verify_cap()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-RO:' @@ -671,7 +677,7 @@ def wrap_dirnode_cap(filecap): raise AssertionError("cannot interpret as a directory cap: %s" % filecap.__class__) -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-Verifier:' @@ -696,12 +702,12 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): return self -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI + INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] def __init__(self, filenode_uri=None): if filenode_uri: diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index a7b77001a..42e2ec5fb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -27,6 +27,7 @@ PORTED_MODULES = [ "allmydata.__main__", "allmydata._auto_deps", "allmydata._monkeypatch", + "allmydata.blacklist", "allmydata.codec", "allmydata.crypto", "allmydata.crypto.aes", @@ -34,6 +35,7 @@ PORTED_MODULES = [ "allmydata.crypto.error", "allmydata.crypto.rsa", "allmydata.crypto.util", + "allmydata.deep_stats", "allmydata.dirnode", "allmydata.hashtree", "allmydata.immutable.checker", @@ -69,6 +71,7 @@ PORTED_MODULES = [ "allmydata.mutable.servermap", "allmydata.node", "allmydata.nodemaker", + "allmydata.stats", "allmydata.storage_client", "allmydata.storage.common", "allmydata.storage.crawler", @@ -80,6 +83,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test.no_network", "allmydata.test.mutable.util", + "allmydata.unknown", "allmydata.uri", "allmydata.util._python3", "allmydata.util.abbreviate", @@ -110,6 +114,8 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.web.logs", + "allmydata.webish", ] PORTED_TEST_MODULES = [ @@ -166,6 +172,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_repairer", "allmydata.test.test_spans", "allmydata.test.test_statistics", + "allmydata.test.test_stats", "allmydata.test.test_storage", "allmydata.test.test_storage_client", "allmydata.test.test_storage_web", @@ -181,6 +188,11 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", - "allmydata.test.web.test_util", + "allmydata.test.web.test_introducer", + "allmydata.test.web.test_logs", + "allmydata.test.web.test_private", + "allmydata.test.web.test_root", "allmydata.test.web.test_status", + "allmydata.test.web.test_util", + "allmydata.test.web.test_webish", ] diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 1d13f61e6..ed2a11ee4 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -15,7 +15,18 @@ if PY2: import time +try: + from typing import ( + Callable, + Any, + ) +except ImportError: + pass + from foolscap.api import eventually +from eliot.twisted import ( + inline_callbacks, +) from twisted.internet import defer, reactor, error from twisted.python.failure import Failure @@ -201,3 +212,22 @@ class WaitForDelayedCallsMixin(PollMixin): d.addErrback(log.err, "error while waiting for delayed calls") d.addBoth(lambda ign: res) return d + +@inline_callbacks +def until( + action, # type: Callable[[], defer.Deferred[Any]] + condition, # type: Callable[[], bool] +): + # type: (...) -> defer.Deferred[None] + """ + Run a Deferred-returning function until a condition is true. + + :param action: The action to run. + :param condition: The predicate signaling stop. + + :return: A Deferred that fires after the condition signals stop. + """ + while True: + yield action() + if condition(): + break diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index ea16c0d6a..e40e06180 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -311,7 +311,7 @@ def precondition_abspath(path): _getfullpathname = None try: - from nt import _getfullpathname + from nt import _getfullpathname # type: ignore except ImportError: pass diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 5d1716853..582bafe86 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -14,6 +14,12 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time + +try: + from typing import List +except ImportError: + pass + from twisted.internet import task class TimeoutError(Exception): @@ -23,7 +29,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] + _poll_should_ignore_these_errors = [] # type: List[Exception] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 3e9eb8379..cde3924fd 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -6,6 +6,11 @@ Can eventually be merged back into allmydata.web.common. from past.builtins import unicode +try: + from typing import Optional +except ImportError: + pass + from twisted.web import resource, http from allmydata.util import abbreviate @@ -55,7 +60,7 @@ class MultiFormatResource(resource.Resource, object): format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" - formatDefault = None + formatDefault = None # type: Optional[str] def render(self, req): """ diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 6ec558e82..280d6cc26 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource): self.introducer_node = introducer_node self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource - self.putChild("", self) + self.putChild(b"", self) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen))) def _create_element(self): """ @@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource): announcement_summary[service_name] += 1 res[u"announcement_summary"] = announcement_summary - return json.dumps(res, indent=1) + b"\n" + return (json.dumps(res, indent=1) + "\n").encode("utf-8") class IntroducerRootElement(Element): diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 0ba8b17e9..6f15a3ca9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -49,7 +52,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - self.sendMessage(json.dumps(message)) + encoded = json.dumps(message) + if isinstance(encoded, str): + # On Python 3 dumps() returns Unicode... + encoded = encoded.encode("utf-8") + self.sendMessage(encoded) def onOpen(self): """ diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index fea058405..405ca75e7 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,16 @@ class IToken(ICredentials): pass -@implementer(IToken) +# Workaround for Shoobx/mypy-zope#26, where without suitable +# stubs for twisted classes (ICredentials), IToken does not +# appear to be an Interface. The proper fix appears to be to +# create stubs for twisted +# (https://twistedmatrix.com/trac/ticket/9717). For now, +# bypassing the inline decorator syntax works around the issue. +_itoken_impl = implementer(IToken) + + +@_itoken_impl @attr.s class Token(object): proposed_token = attr.ib(type=bytes) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index fdc72ab71..0ef6b00d2 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,6 +1,8 @@ +from future.utils import PY3 + import os import time -import urllib +from urllib.parse import quote as urlquote from hyperlink import DecodedURL, URL from pkg_resources import resource_filename @@ -81,7 +83,7 @@ class URIHandler(resource.Resource, object): # it seems Nevow was creating absolute URLs including # host/port whereas req.uri is absolute (but lacks host/port) redir_uri = URL.from_text(req.prePathURL().decode('utf8')) - redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) + redir_uri = redir_uri.child(urlquote(uri_arg)) # add back all the query args that AREN'T "?uri=" for k, values in req.args.items(): if k != b"uri": @@ -227,26 +229,26 @@ class Root(MultiFormatResource): self._client = client self._now_fn = now_fn - # Children need to be bytes; for now just doing these to make specific - # tests pass on Python 3, but eventually will do all them when this - # module is ported to Python 3 (if not earlier). self.putChild(b"uri", URIHandler(client)) - self.putChild("cap", URIHandler(client)) + self.putChild(b"cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource # hierarchy which is only accessible with the private per-node API # auth token. - self.putChild("private", create_private_tree(client.get_auth_token)) + self.putChild(b"private", create_private_tree(client.get_auth_token)) - self.putChild("file", FileHandler(client)) - self.putChild("named", FileHandler(client)) - self.putChild("status", status.Status(client.get_history())) - self.putChild("statistics", status.Statistics(client.stats_provider)) + self.putChild(b"file", FileHandler(client)) + self.putChild(b"named", FileHandler(client)) + self.putChild(b"status", status.Status(client.get_history())) + self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + child_path = filen + if PY3: + child_path = filen.encode("utf-8") + self.putChild(child_path, static.File(os.path.join(static_dir, filen))) - self.putChild("report_incident", IncidentReporter()) + self.putChild(b"report_incident", IncidentReporter()) @exception_to_child def getChild(self, path, request): diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index f32f56714..e90fa573a 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str import re, time, tempfile @@ -65,18 +77,24 @@ class TahoeLAFSRequest(Request, object): self.path, argstring = x self.args = parse_qs(argstring, 1) - if self.method == 'POST': + if self.method == b'POST': # We use FieldStorage here because it performs better than # cgi.parse_multipart(self.content, pdict) which is what # twisted.web.http.Request uses. - self.fields = FieldStorage( - self.content, - { - name.lower(): value[-1] - for (name, value) - in self.requestHeaders.getAllRawHeaders() - }, - environ={'REQUEST_METHOD': 'POST'}) + + headers = { + ensure_str(name.lower()): ensure_str(value[-1]) + for (name, value) + in self.requestHeaders.getAllRawHeaders() + } + + if 'content-length' not in headers: + # Python 3's cgi module would really, really like us to set Content-Length. + self.content.seek(0, 2) + headers['content-length'] = str(self.content.tell()) + self.content.seek(0) + + self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'}) self.content.seek(0) self._tahoeLAFSSecurityPolicy() diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e7f045b95..e98aa8a67 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -217,7 +217,12 @@ def initialize(): # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): - return re.sub(u'\\x7F[0-9a-fA-F]*\\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), s) + return re.sub( + u'\\x7F[0-9a-fA-F]*\\;', + # type ignored for 'unichr' (Python 2 only) + lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore + s, + ) try: argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)] diff --git a/tox.ini b/tox.ini index d32f90b15..22e55f267 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,pypy27 minversion = 2.4 [testenv] @@ -112,6 +112,16 @@ commands = # file. See pyproject.toml for legal values. python -m towncrier.check --pyproject towncrier.pyproject.toml + +[testenv:typechecks] +skip_install = True +deps = + mypy + git+https://github.com/Shoobx/mypy-zope + git+https://github.com/warner/foolscap +commands = mypy src + + [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi"