diff --git a/.circleci/config.yml b/.circleci/config.yml index b00bcdcec..28e4c8d58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,79 +15,66 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - # Every job that pulls a Docker image from Docker Hub needs to provide - # credentials for that pull operation to avoid being subjected to - # unauthenticated pull limits shared across all of CircleCI. Use this - # first job to define a yaml anchor that can be used to supply a - # CircleCI job context which makes Docker Hub credentials available in - # the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "debian-9": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" - + - "debian-9": + {} - "debian-10": - <<: *DOCKERHUB_CONTEXT requires: - "debian-9" - "ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT + {} - "ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "ubuntu-16-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "fedora-29": - <<: *DOCKERHUB_CONTEXT + {} - "fedora-28": - <<: *DOCKERHUB_CONTEXT requires: - "fedora-29" - "centos-8": - <<: *DOCKERHUB_CONTEXT + {} - "nixos-19-09": - <<: *DOCKERHUB_CONTEXT + {} # Test against PyPy 2.7 - "pypy27-buster": - <<: *DOCKERHUB_CONTEXT + {} # Just one Python 3.6 configuration while the port is in-progress. - "python36": - <<: *DOCKERHUB_CONTEXT + {} # Other assorted tasks and configurations - "lint": - <<: *DOCKERHUB_CONTEXT + {} + - "codechecks3": + {} - "pyinstaller": - <<: *DOCKERHUB_CONTEXT + {} - "deprecations": - <<: *DOCKERHUB_CONTEXT + {} - "c-locale": - <<: *DOCKERHUB_CONTEXT + {} # Any locale other than C or UTF-8. - "another-locale": - <<: *DOCKERHUB_CONTEXT + {} - "integration": - <<: *DOCKERHUB_CONTEXT requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - "debian-9" - "typechecks": - <<: *DOCKERHUB_CONTEXT + {} + - "docs": + {} images: # Build the Docker images used by the ci jobs. This makes the ci jobs @@ -102,8 +89,16 @@ workflows: - "master" jobs: - - "build-image-debian-10": - <<: *DOCKERHUB_CONTEXT + # Every job that pushes a Docker image from Docker Hub needs to provide + # credentials. Use this first job to define a yaml anchor that can be + # used to supply a CircleCI job context which makes Docker Hub + # credentials available in the environment. + # + # Contexts are managed in the CircleCI web interface: + # + # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" - "build-image-debian-9": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-16-04": @@ -165,6 +160,24 @@ jobs: command: | ~/.local/bin/tox -e codechecks + codechecks3: + docker: + - <<: *DOCKERHUB_AUTH + image: "circleci/python:3" + + steps: + - "checkout" + + - run: + name: "Install tox" + command: | + pip install --user tox + + - run: + name: "Static-ish code checks" + command: | + ~/.local/bin/tox -e codechecks3 + pyinstaller: docker: - <<: *DOCKERHUB_AUTH @@ -458,6 +471,18 @@ jobs: command: | /tmp/venv/bin/tox -e typechecks + docs: + docker: + - <<: *DOCKERHUB_AUTH + image: "tahoelafsci/ubuntu:18.04-py3" + + steps: + - "checkout" + - run: + name: "Build documentation" + command: | + /tmp/venv/bin/tox -e docs + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e14268f23..e95d2ee88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,22 +18,22 @@ jobs: fail-fast: false matrix: os: - - macos-latest - windows-latest + - ubuntu-latest python-version: - 2.7 + - 3.6 + - 3.7 + - 3.8 + - 3.9 + include: + # On macOS don't bother with 3.6-3.8, just to get faster builds. + - os: macos-latest + python-version: 2.7 + - os: macos-latest + python-version: 3.9 steps: - - # Get vcpython27 on Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - # See https://github.com/actions/checkout. A fetch-depth of 0 # fetches all tags and branches. - name: Check out Tahoe-LAFS sources @@ -42,7 +42,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -67,14 +67,14 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools + pip install --upgrade codecov tox tox-gh-actions setuptools pip list - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e py27-coverage" - run: tox -e py27-coverage + - name: Run tox for corresponding Python version + run: python -m tox - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 @@ -87,10 +87,29 @@ jobs: # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the # coveralls.io-supplied Python reporter, for this. + # + # It is coveralls-python 1.x that has maintained compatibility + # with Python 2, while coveralls-python 3.x is compatible with + # Python 3. Sadly we can't use them both in the same workflow. + # + # The two versions of coveralls-python are somewhat mutually + # incompatible. Mixing these two different versions when + # reporting coverage to coveralls.io will lead to grief, since + # they get job IDs in different fashion. If we use both + # versions of coveralls in the same workflow, the finalizing + # step will be able to mark only part of the jobs as done, and + # the other part will be left hanging, never marked as done: it + # does not matter if we make an API call or `coveralls --finish` + # to indicate that CI has finished running. + # + # So we try to use the newer coveralls-python that is available + # via Python 3 (which is present in GitHub Actions tool cache, + # even when we're running Python 2.7 tests) throughout this + # workflow. - name: "Report Coverage to Coveralls" run: | - pip install coveralls - python -m coveralls + pip3 install --upgrade coveralls==3.0.1 + python3 -m coveralls env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -113,80 +132,22 @@ jobs: # a single report, we have to tell Coveralls when we've uploaded all of the # data files. This does it. We make sure it runs last by making it depend # on *all* of the coverage-collecting jobs. + # + # See notes about parallel builds on GitHub Actions at + # https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html finish-coverage-report: - # There happens to just be one coverage-collecting job at the moment. If - # the coverage reports are broken and someone added more - # coverage-collecting jobs to this workflow but didn't update this, that's - # why. - needs: + needs: - "coverage" runs-on: "ubuntu-latest" + container: "python:3-slim" steps: - - name: "Check out Tahoe-LAFS sources" - uses: "actions/checkout@v2" - - - name: "Finish Coveralls Reporting" + - name: "Indicate completion to coveralls.io" run: | - # coveralls-python does have a `--finish` option but it doesn't seem - # to work, at least for us. - # https://github.com/coveralls-clients/coveralls-python/issues/248 - # - # But all it does is this simple POST so we can just send it - # ourselves. The only hard part is guessing what the POST - # parameters mean. And I've done that for you already. - # - # Since the build is done I'm going to guess that "done" is a fine - # value for status. - # - # That leaves "build_num". The coveralls documentation gives some - # hints about it. It suggests using $CIRCLE_WORKFLOW_ID if your job - # is on CircleCI. CircleCI documentation says this about - # CIRCLE_WORKFLOW_ID: - # - # Observation of the coveralls.io web interface, logs from the - # coveralls command in action, and experimentation suggests the - # value for PRs is something more like: - # - # -PR- - # - # For branches, it's just the git branch tip hash. - - # For pull requests, refs/pull//merge was just checked out - # by so HEAD will refer to the right revision. For branches, HEAD - # is also the tip of the branch. - REV=$(git rev-parse HEAD) - - # We can get the PR number from the "context". - # - # https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request - # - # (via ). - # - # If this is a pull request, `github.event` is a `pull_request` - # structure which has `number` right in it. - # - # If this is a push, `github.event` is a `push` instead but we only - # need the revision to construct the build_num. - - PR=${{ github.event.number }} - - if [ "${PR}" = "" ]; then - BUILD_NUM=$REV - else - BUILD_NUM=$REV-PR-$PR - fi - REPO_NAME=$GITHUB_REPOSITORY - - curl \ - -k \ - https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \ - -d \ - "payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME" + pip3 install --upgrade coveralls==3.0.1 + python3 -m coveralls --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - # Help coveralls identify our project. - COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" integration: runs-on: ${{ matrix.os }} @@ -194,29 +155,34 @@ jobs: fail-fast: false matrix: os: - - macos-latest - windows-latest + - ubuntu-latest python-version: - 2.7 + - 3.6 + - 3.9 + include: + # On macOS don't bother with 3.6, just to get faster builds. + - os: macos-latest + python-version: 2.7 + - os: macos-latest + python-version: 3.9 steps: - # Get vcpython27 for Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - - name: Install Tor [Ubuntu] if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - - - name: Install Tor [macOS] - if: matrix.os == 'macos-latest' - run: brew install tor + + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. + # We have to use an older version of Tor for running integration + # tests on macOS. + - name: Install Tor [macOS, ${{ matrix.python-version }} ] + if: ${{ matrix.os == 'macos-latest' }} + run: | + brew extract --version 0.4.5.8 tor homebrew/cask + brew install tor@0.4.5.8 + brew link --overwrite tor@0.4.5.8 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' @@ -255,9 +221,14 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e integration" + - name: Run "Python 2 integration tests" + if: ${{ matrix.python-version == '2.7' }} run: tox -e integration + - name: Run "Python 3 integration tests" + if: ${{ matrix.python-version != '2.7' }} + run: tox -e integration3 + - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 if: failure() @@ -279,15 +250,6 @@ jobs: steps: - # Get vcpython27 for Windows + Python 2.7, to build netifaces - # extension. See https://chocolatey.org/packages/vcpython27 and - # https://github.com/crazy-max/ghaction-chocolatey - - name: Install MSVC 9.0 for Python 2.7 [Windows] - if: matrix.os == 'windows-latest' && matrix.python-version == '2.7' - uses: crazy-max/ghaction-chocolatey@v1 - with: - args: install vcpython27 - - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 with: diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index efc2479ca..000000000 --- a/.lgtm.yml +++ /dev/null @@ -1,22 +0,0 @@ -extraction: - python: - after_prepare: - - | - # https://discuss.lgtm.com/t/determination-of-python-requirements/974/4 - sed -i 's/\("pyOpenSSL\)/\# Dependency removed for lgtm (see .lgtm.yml): \1/g' src/allmydata/_auto_deps.py - -queries: - # This generates spurious errors for calls by interface because of the - # zope.interface choice to exclude self from method signatures. So, turn it - # off. - - exclude: "py/call/wrong-arguments" - - # The premise of this query is broken. The errors it produces are nonsense. - # There is no such thing as a "procedure" in Python and "None" is not - # meaningless. - - exclude: "py/procedure-return-value-used" - - # It is true that this query identifies things which are sometimes mistakes. - # However, it also identifies things which are entirely valid. Therefore, - # it produces noisy results. - - exclude: "py/implicit-string-concatenation-in-list" \ No newline at end of file diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst new file mode 100644 index 000000000..70d308398 --- /dev/null +++ b/CONTRIBUTORS.rst @@ -0,0 +1,42 @@ +Contributor Checklist +===================== + + +* Create a ``Trac`` ticket, fill it out and assign it to yourself (contact exarkun if you don't have an account): + + ``https://tahoe-lafs.org/trac/tahoe-lafs/newticket`` + +* Use the ticket number to name your branch (example): + + ``3003.contributor-guide`` + +* Good idea to add tests at the same time you write your code. + +* Add a file to the ``/newsfragments`` folder, named with the ticket number and the type of patch (example): + + ``newsfragments/3651.minor`` + +* ``towncrier`` recognizes the following types: + + ``incompat``, ``feature``, ``bugfix``, ``installation``, ``configuration``, ``documentation``, ``removed``, ``other``, ``minor`` +* Add one sentence to ``newsfragments/.`` describing the change (example): + + ``The integration test suite has been updated to use pytest-twisted instead of deprecated pytest APIs.`` + +* Run the test suite with ``tox``, ``tox -e codechecks`` and ``tox -e typechecks`` + +* Push your branch to Github with your ticket number in the merge commit message (example): + + ``Fixes ticket:3003`` + + This makes the ``Trac`` ticket close when your PR gets approved. + +* Request appropriate review - we suggest asking `Tahoe Committers `__ + +References +---------- + +This checklist is a summary of `this page on contributing Patches `__ + +Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ +and the `Contributor Code of Conduct `__. diff --git a/CREDITS b/CREDITS index 07ac1e476..b0923fc35 100644 --- a/CREDITS +++ b/CREDITS @@ -204,6 +204,34 @@ E: meejah@meejah.ca P: 0xC2602803128069A7, 9D5A 2BD5 688E CB88 9DEB CD3F C260 2803 1280 69A7 D: various bug-fixes and features +N: Chad Whitacre +E: chad@zetaweb.com +D: Python3 porting + +N: Itamar Turner-Trauring +E: itamar@pythonspeed.com +D: Python3 porting + +N: Jason R. Coombs +E: jaraco@jaraco.com +D: Python3 porting + +N: Maciej Fijalkowski +E: fijall@gmail.com +D: Python3 porting + +N: Ross Patterson +E: me@rpatterson.net +D: Python3 porting + +N: Sajith Sasidharan +E: sajith@hcoop.net +D: Python3 porting + +N: Pete Fein +E: pete@snake.dev +D: Python3 porting + N: Viktoriia Savchuk W: https://twitter.com/viktoriiasvchk D: Developer community focused improvements on the README file. diff --git a/NEWS.rst b/NEWS.rst index 2ca67a1f6..88d231826 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -5,6 +5,61 @@ User-Visible Changes in Tahoe-LAFS ================================== .. towncrier start line +Release 1.15.1 +'''''''''''''' + +Misc/Other +---------- + +- `#3469 `_, `#3608 `_ + + +Release 1.15.0 (2020-10-13) +''''''''''''''''''''''''''' + +Features +-------- + +- PyPy is now a supported platform. (`#1792 `_) +- allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. (`#3317 `_) + + +Bug Fixes +--------- + +- Make directory page links work. (`#3312 `_) +- Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests. (`#3348 `_) +- Mutable files now use RSA exponent 65537 (`#3349 `_) + + +Dependency/Installation Changes +------------------------------- + +- Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7. (`#3296 `_) +- Tahoe-LAFS now supports Ubuntu 20.04. (`#3328 `_) + + +Removed Features +---------------- + +- The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at . (`#3284 `_) +- Slackware 14.2 is no longer a Tahoe-LAFS supported platform. (`#3323 `_) + + +Other Changes +------------- + +- The Tahoe-LAFS project has adopted a formal code of conduct. (`#2755 `_) +- (`#3263 `_, `#3324 `_) +- The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments. (`#3355 `_) + + +Misc/Other +---------- + +- `#3247 `_, `#3254 `_, `#3277 `_, `#3278 `_, `#3287 `_, `#3288 `_, `#3289 `_, `#3290 `_, `#3291 `_, `#3292 `_, `#3293 `_, `#3294 `_, `#3297 `_, `#3298 `_, `#3299 `_, `#3300 `_, `#3302 `_, `#3303 `_, `#3304 `_, `#3305 `_, `#3306 `_, `#3308 `_, `#3309 `_, `#3313 `_, `#3315 `_, `#3316 `_, `#3320 `_, `#3325 `_, `#3326 `_, `#3329 `_, `#3330 `_, `#3331 `_, `#3332 `_, `#3333 `_, `#3334 `_, `#3335 `_, `#3336 `_, `#3338 `_, `#3339 `_, `#3340 `_, `#3341 `_, `#3342 `_, `#3343 `_, `#3344 `_, `#3346 `_, `#3351 `_, `#3353 `_, `#3354 `_, `#3356 `_, `#3357 `_, `#3358 `_, `#3359 `_, `#3361 `_, `#3364 `_, `#3365 `_, `#3366 `_, `#3367 `_, `#3368 `_, `#3370 `_, `#3372 `_, `#3373 `_, `#3374 `_, `#3375 `_, `#3376 `_, `#3377 `_, `#3378 `_, `#3380 `_, `#3381 `_, `#3382 `_, `#3383 `_, `#3386 `_, `#3387 `_, `#3388 `_, `#3389 `_, `#3391 `_, `#3392 `_, `#3393 `_, `#3394 `_, `#3395 `_, `#3396 `_, `#3397 `_, `#3398 `_, `#3401 `_, `#3403 `_, `#3406 `_, `#3408 `_, `#3409 `_, `#3411 `_, `#3415 `_, `#3416 `_, `#3417 `_, `#3421 `_, `#3422 `_, `#3423 `_, `#3424 `_, `#3425 `_, `#3426 `_, `#3427 `_, `#3429 `_, `#3430 `_, `#3431 `_, `#3436 `_, `#3437 `_, `#3438 `_, `#3439 `_, `#3440 `_, `#3442 `_, `#3443 `_, `#3446 `_, `#3448 `_, `#3449 `_, `#3450 `_, `#3451 `_, `#3452 `_, `#3453 `_, `#3455 `_, `#3456 `_, `#3458 `_, `#3462 `_, `#3463 `_, `#3464 `_ + + Release 1.14.0 (2020-03-11) ''''''''''''''''''''''''''' diff --git a/README.rst b/README.rst index b1f6d2563..2cc6e38eb 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ Free and Open decentralized data store `Tahoe-LAFS `__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security. -|Contributor Covenant| |readthedocs| |travis| |circleci| |coveralls| +|Contributor Covenant| |readthedocs| |circleci| |githubactions| |coveralls| Table of contents @@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site `__. +- Chat with Tahoe-LAFS developers at ``#tahoe-lafs`` channel on `libera.chat `__ IRC network or `Slack `__. - Join our `weekly conference calls `__ with core developers and interested community members. @@ -93,6 +93,10 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard `__ and the `Contributor Code of Conduct `__. +🤝 Supporters +-------------- + +We would like to thank `Fosshost `__ for supporting us with hosting services. If your open source project needs help, you can apply for their support. ❓ FAQ ------ @@ -118,13 +122,12 @@ See `TGPPL.PDF `__ for why the TGPPL ex :alt: documentation status :target: http://tahoe-lafs.readthedocs.io/en/latest/?badge=latest -.. |travis| image:: https://travis-ci.org/tahoe-lafs/tahoe-lafs.png?branch=master - :alt: build status - :target: https://travis-ci.org/tahoe-lafs/tahoe-lafs - .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs +.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/ci.yml/badge.svg + :target: https://github.com/tahoe-lafs/tahoe-lafs/actions + .. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg :alt: code coverage :target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 3b14f2d70..174157e8c 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -45,6 +45,7 @@ The following community members have made themselves available for conduct issue - Jean-Paul Calderone (jean-paul at leastauthority dot com) - meejah (meejah at meejah dot ca) +- May-Lee Sia(she/her) (tahoe dot lafs dot community at gmail dot com) This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.3.0, available at diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst deleted file mode 100644 index e47d87bd6..000000000 --- a/docs/INSTALL.rst +++ /dev/null @@ -1,343 +0,0 @@ -.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- - -.. - note: if you aren't reading the rendered form of these docs at - http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any - ":doc:" links refer to other files in this docs/ directory - -********************* -Installing Tahoe-LAFS -********************* - -Welcome to `the Tahoe-LAFS project`_, a secure, decentralized, fault-tolerant -storage system. See :doc:`about` for an overview of the architecture and -security properties of the system. - -This procedure should work on Windows, Mac, illumos (previously OpenSolaris), -and too many flavors of Linux and of BSD to list. - -.. _the Tahoe-LAFS project: https://tahoe-lafs.org - -First: In Case Of Trouble -========================= - -In some cases these instructions may fail due to peculiarities of your -platform. - -If the following instructions don't Just Work without any further effort on -your part, then please write to `the tahoe-dev mailing list`_ where friendly -hackers will help you out. - -.. _the tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev - -Pre-Packaged Versions -===================== - -You may not need to build Tahoe at all. - -If you are on Windows, please see :doc:`windows` for platform-specific -instructions. - -If you are on a Mac, you can either follow these instructions, or use the -pre-packaged bundle described in :doc:`OS-X`. - -Many Linux distributions include Tahoe-LAFS packages. Debian and Ubuntu users -can ``apt-get install tahoe-lafs``. See `OSPackages`_ for other -platforms. - -.. _OSPackages: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/OSPackages - - -Preliminaries -============= - -If you don't use a pre-packaged copy of Tahoe, you can build it yourself. -You'll need Python2.7, pip, and virtualenv. -Tahoe-LAFS depends on some libraries which require a C compiler to build. -However, for many platforms, PyPI hosts already-built packages of libraries. - -If there is no already-built package for your platform, -you will need a C compiler, -the Python development headers, -and some libraries (libffi-dev and libssl-dev). - -On a modern Debian/Ubuntu-derived distribution, this command will get you -everything you need:: - - apt-get install build-essential python-dev libffi-dev libssl-dev libyaml-dev python-virtualenv - -On OS-X, install pip and virtualenv as described below. If you want to -compile the dependencies yourself, you'll also need to install -Xcode and its command-line tools. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - -Python 2.7 ----------- - -Check if you already have an adequate version of Python installed by running -``python -V``. The latest version of Python v2.7 is recommended, which is -2.7.11 as of this writing. Python v2.6.x and v3 do not work. On Windows, we -recommend the use of native Python v2.7, not Cygwin Python. If you don't have -one of these versions of Python installed, `download`_ and install the latest -version of Python v2.7. Make sure that the path to the installation directory -has no spaces in it (e.g. on Windows, do not install Python in the "Program -Files" directory):: - - % python --version - Python 2.7.11 - -.. _download: https://www.python.org/downloads/ - -pip ---- - -Many Python installations already include ``pip``, but in case yours does -not, get it with the `pip install instructions`_:: - - % pip --version - pip 10.0.1 from ... (python 2.7) - -.. _pip install instructions: https://pip.pypa.io/en/stable/installing/ - -virtualenv ----------- - -If you do not have an OS-provided copy of ``virtualenv``, install it with the -instructions from the `virtualenv documentation`_:: - - - % virtualenv --version - 15.1.0 - -.. _virtualenv documentation: https://virtualenv.pypa.io/en/latest/installation.html - -C compiler and libraries ------------------------- - -Except on OS-X, where the Tahoe project hosts pre-compiled wheels for all -dependencies, you will need several C libraries installed before you can -build. You will also need the Python development headers, and a C compiler -(your python installation should know how to find these). - -On Debian/Ubuntu-derived systems, the necessary packages are ``python-dev``, -``libffi-dev``, and ``libssl-dev``, and can be installed with ``apt-get``. On -RPM-based system (like Fedora) these may be named ``python-devel``, etc, -instead, and cam be installed with ``yum`` or ``rpm``. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - - -Install the Latest Tahoe-LAFS Release -===================================== - -We recommend creating a fresh virtualenv for your Tahoe-LAFS install, to -isolate it from any python packages that are already installed (and to -isolate the rest of your system from Tahoe's dependencies). - -This example uses a virtualenv named ``venv``, but you can call it anything -you like. Many people prefer to keep all their virtualenvs in one place, like -``~/.local/venvs/`` or ``~/venvs/``. - -It's usually a good idea to upgrade the virtualenv's ``pip`` and -``setuptools`` to their latest versions, with ``venv/bin/pip install -U pip -setuptools``. Many operating systems have an older version of ``virtualenv``, -which then includes older versions of pip and setuptools. Upgrading is easy, -and only affects the virtualenv: not the rest of your computer. - -Then use the virtualenv's ``pip`` to install the latest Tahoe-LAFS release -from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run -``venv/bin/tahoe --version`` to confirm the install was successful:: - - % virtualenv venv - New python executable in ~/venv/bin/python2.7 - Installing setuptools, pip, wheel...done. - - % venv/bin/pip install -U pip setuptools - Downloading/unpacking pip from https://pypi.python.org/... - ... - Successfully installed pip setuptools - - % venv/bin/pip install tahoe-lafs - Collecting tahoe-lafs - ... - Installing collected packages: ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.14.0 - foolscap: ... - - % - -Install From a Source Tarball ------------------------------ - -You can also install directly from the source tarball URL. To verify -signatures, first see verifying_signatures_ and replace the URL in the -following instructions with the local filename. - - % virtualenv venv - New python executable in ~/venv/bin/python2.7 - Installing setuptools, pip, wheel...done. - - % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 - Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2 - ... - Installing collected packages: ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.14.0 - ... - -.. _verifying_signatures: - -Verifying Signatures --------------------- - -First download the source tarball and then any signatures. There are several -developers who are able to produce signatures for a release. A release may -have multiple signatures. All should be valid and you should confirm at least -one of them (ideally, confirm all). - -This statement, signed by the existing Tahoe release-signing key, attests to -those developers authorized to sign a Tahoe release: - -.. include:: developer-release-signatures - :code: - -Signatures are made available beside the release. So for example, a release -like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might -have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and -``tahoe-lafs-1.16.0.tar.bz2.warner.asc``. - -To verify the signatures using GnuPG:: - - % gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2 - gpg: Signature made XXX - gpg: using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7 - gpg: Good signature from "meejah " [full] - % gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2 - gpg: Signature made XXX - gpg: using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF - gpg: Good signature from "Brian Warner " [full] - - - -Extras ------- - -Tahoe-LAFS provides some functionality only when explicitly requested at installation time. -It does this using the "extras" feature of setuptools. -You can request these extra features when running the ``pip install`` command like this:: - - % venv/bin/pip install tahoe-lafs[tor] - -This example enables support for listening and connecting using Tor. -The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested. - -Hacking On Tahoe-LAFS ---------------------- - -To modify the Tahoe source code, you should get a git checkout, and install -with the ``--editable`` flag. You should also use the ``[test]`` extra to get -the additional libraries needed to run the unit tests:: - - % git clone https://github.com/tahoe-lafs/tahoe-lafs.git - - % cd tahoe-lafs - - % virtualenv venv - - % venv/bin/pip install --editable .[test] - Obtaining file::~/tahoe-lafs - ... - Successfully installed ... - - % venv/bin/tahoe --version - tahoe-lafs: 1.14.0.post34.dev0 - ... - -This way, you won't have to re-run the ``pip install`` step each time you -modify the source code. - -Running the ``tahoe`` executable -================================ - -The rest of the Tahoe-LAFS documentation assumes that you can run the -``tahoe`` executable that you just created. You have four basic options: - -* Use the full path each time (e.g. ``~/venv/bin/tahoe``). -* "`Activate`_" the virtualenv with ``. venv/bin/activate``, to get a - subshell with a ``$PATH`` that includes the ``venv/bin/`` directory, then - you can just run ``tahoe``. -* Change your ``$PATH`` to include the ``venv/bin/`` directory, so you can - just run ``tahoe``. -* Symlink from ``~/bin/tahoe`` to the ``tahoe`` executable. Since ``~/bin`` - is typically in your ``$PATH`` (at least if it exists when you log in), - this will let you just run ``tahoe``. - -You might also find the `pipsi`_ tool convenient: ``pipsi install -tahoe-lafs`` will create a new virtualenv, install tahoe into it, then -symlink just the executable (into ``~/.local/bin/tahoe``). Then either add -``~/.local/bin/`` to your ``$PATH``, or make one last symlink into -``~/bin/tahoe``. - -.. _Activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script -.. _pipsi: https://pypi.python.org/pypi/pipsi/0.9 - -Running the Self-Tests -====================== - -To run the self-tests from a source tree, you'll need ``tox`` installed. On a -Debian/Ubuntu system, use ``apt-get install tox``. You can also install it -into your tahoe-specific virtualenv with ``pip install tox``. - -Then just run ``tox``. This will create a new fresh virtualenv, install Tahoe -(from the source tree, including any changes you have made) and all its -dependencies (including testing-only dependencies) into the virtualenv, then -run the unit tests. This ensures that the tests are repeatable and match the -results of other users, unaffected by any other Python packages installed on -your machine. On a modern computer this will take 5-10 minutes, and should -result in a "all tests passed" mesage:: - - % tox - GLOB sdist-make: ~/tahoe-lafs/setup.py - py27 recreate: ~/tahoe-lafs/.tox/py27 - py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.14.0.post8.dev0.zip - py27 runtests: commands[0] | tahoe --version - py27 runtests: commands[1] | trial --rterrors allmydata - allmydata.test.test_auth - AccountFileCheckerKeyTests - test_authenticated ... [OK] - test_missing_signature ... [OK] - ... - Ran 1186 tests in 423.179s - - PASSED (skips=7, expectedFailures=3, successes=1176) - __________________________ summary ___________________________________ - py27: commands succeeded - congratulations :) - -Common Problems -=============== - -If you see an error like ``fatal error: Python.h: No such file or directory`` -while compiling the dependencies, you need the Python development headers. If -you are on a Debian or Ubuntu system, you can install them with ``sudo -apt-get install python-dev``. On RedHat/Fedora, install ``python-devel``. - -Similar errors about ``openssl/crypto.h`` indicate that you are missing the -OpenSSL development headers (``libssl-dev``). Likewise ``ffi.h`` means you -need ``libffi-dev``. - -**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater. - - -Using Tahoe-LAFS -================ - -Now you are ready to deploy a decentralized filesystem. You will use the -``tahoe`` executable to create, configure, and launch your Tahoe-LAFS nodes. -See :doc:`running` for instructions on how to do that. diff --git a/docs/desert-island.rst b/docs/Installation/install-on-desert-island.rst similarity index 97% rename from docs/desert-island.rst rename to docs/Installation/install-on-desert-island.rst index 33db243d6..fbc59ddbd 100644 --- a/docs/desert-island.rst +++ b/docs/Installation/install-on-desert-island.rst @@ -1,6 +1,6 @@ -****************************************** -How To Build Tahoe-LAFS On A Desert Island -****************************************** +*************************************** +Building Tahoe-LAFS On A Desert Island +*************************************** (or an airplane, or anywhere else without internet connectivity) diff --git a/docs/Installation/install-on-linux.rst b/docs/Installation/install-on-linux.rst new file mode 100644 index 000000000..79fa1c066 --- /dev/null +++ b/docs/Installation/install-on-linux.rst @@ -0,0 +1,75 @@ +**************************** +Building Tahoe-LAFS on Linux +**************************** + +Tahoe-LAFS has made packages available for installing on many linux and BSD distributions. +Debian and Ubuntu users can use ``apt-get install tahoe-lafs``. +If you are working on a Linux distribution which does not have Tahoe-LAFS or are looking to hack on the source code, you can build Tahoe-LAFS yourself: + +Prerequisites +============= + +Make sure the following are installed: + +* **Python 3's latest version**: Check for the version by running ``python --version``. +* **pip**: Most python installations already include ``pip``. However, if your installation does not, see `pip installation `_. +* **virtualenv**: Use ``pip`` to install virtualenv:: + + pip install --user virtualenv + +* **C compiler and libraries**: + + * ``python-dev``: Python development headers. + * ``libffi-dev``: Foreign Functions Interface library. + * ``libssl-dev``: SSL library, Tahoe-LAFS needs OpenSSL version 1.1.1c or greater. + + .. note:: + If you are working on Debian or Ubuntu, you can install the necessary libraries using ``apt-get``:: + + apt-get install python-dev libffi-dev libssl-dev + + On an RPM-based system such as Fedora, you can install the necessary libraries using ``yum`` or ``rpm``. However, the packages may be named differently. + +Install the Latest Tahoe-LAFS Release +===================================== + +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS directly from source by creating a ``virtualenv`` instance: + +1. Clone the Tahoe-LAFS repository:: + + git clone https://github.com/tahoe-lafs/tahoe-lafs.git + +2. Move into the tahoe-lafs directory:: + + cd tahoe-lafs + +3. Create a fresh virtualenv for your Tahoe-LAFS install:: + + virtualenv venv + +.. note:: + venv is the name of the virtual environment in this example. Use any name for your environment. + +4. Upgrade ``pip`` and ``setuptools`` on the newly created virtual environment:: + + venv/bin/pip install -U pip setuptools + +5. If you'd like to modify the Tahoe source code, you need to install Tahoe-LAFS with the ``--editable`` flag with the ``test`` extra:: + + venv/bin/pip install --editable .[test] + +.. note:: + Tahoe-LAFS provides extra functionality when requested explicitly at installation using the "extras" feature of setuptools. To learn more about the extras which Tahoe supports, see Tahoe extras. + +6. Verify installation by checking for the version:: + + venv/bin/tahoe --version + +If you do not want to use the full path, i.e., ``venv/bin/tahoe`` everytime you want to run tahoe, you can activate the ``virtualenv``:: + + . venv/bin/activate + + This will generate a subshell with a ``$PATH`` that includes the ``venv/bin/`` directory. + + + diff --git a/docs/Installation/install-on-windows.rst b/docs/Installation/install-on-windows.rst new file mode 100644 index 000000000..5f836df06 --- /dev/null +++ b/docs/Installation/install-on-windows.rst @@ -0,0 +1,45 @@ +****************************** +Building Tahoe-LAFS on Windows +****************************** + +If you are looking to hack on the source code or run pre-release code, we recommend you create a virtualenv instance and install Tahoe-LAFS into that: + + +1. Make sure you have Powershell installed. See `PowerShell installation `_. + +2. Install the latest version of Python 3. Download the .exe file at the `python website `_. + +3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. + +4. Start PowerShell and enter the following command to verify python installation:: + + python --version + +5. Use ``pip`` to install ``virtualenv``:: + + pip install --user virtualenv + +6. Create a fresh virtualenv for your Tahoe-LAFS install using the following command:: + + virtualenv venv + + .. note:: + venv is the name of the virtual environment in this example. Use any name for your environment. + +7. Use pip to install Tahoe-LAFS in the virtualenv instance:: + + venv\Scripts\pip install tahoe-lafs + +6. Verify installation by checking for the version:: + + venv\Scripts\tahoe --version + +If you do not want to use the full path, i.e. ``venv\Scripts\tahoe`` everytime you want to run tahoe, you can: + +* Activate the virtualenv:: + + . venv\Scripts\activate + + This will generate a subshell with a ``$PATH`` that includes the ``venv\Scripts\`` directory. + +* Change your ``$PATH`` to include the ``venv\Scripts`` directory. \ No newline at end of file diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst new file mode 100644 index 000000000..c8b0b521e --- /dev/null +++ b/docs/Installation/install-tahoe.rst @@ -0,0 +1,68 @@ +.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*- + +.. + note: if you aren't reading the rendered form of these docs at + http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any + ":doc:" links refer to other files in this docs/ directory + +********************* +Installing Tahoe-LAFS +********************* + +`Tahoe-LAFS`_ is a secure, decentralized, and fault-tolerant storage system. +To see an overview of the architecture and security properties, see :doc:`Welcome to Tahoe LAFS! <../about-tahoe>` + +Tahoe-LAFS can be installed and used on any of the following operating systems. + +.. _Tahoe-LAFS: https://tahoe-lafs.org + +Microsoft Windows +================= + +To install Tahoe-LAFS on Windows: + +1. Make sure you have Powershell installed. See `PowerShell installation `_. + +2. Install the latest version of Python 3. Download the .exe file at the `python website `_. + +3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**. + +4. Start PowerShell and enter the following command to verify python installation:: + + python --version + +5. Enter the following command to install Tahoe-LAFS:: + + pip install tahoe-lafs + +6. Verify installation by checking for the version:: + + tahoe --version + +If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`. + +Linux, BSD, or MacOS +==================== + +Tahoe-LAFS can be installed on MacOS, many Linux and BSD distributions. If you are using Ubuntu or Debian, run the following command to install Tahoe-LAFS:: + + apt-get install tahoe-lafs + +If you are working on MacOS or a Linux distribution which does not have Tahoe-LAFS packages, you can build it yourself: + +1. Make sure the following are installed: + + * **Python 3's latest version**: Check for the version by running ``python --version``. + * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation `_. + +2. Install Tahoe-LAFS using pip:: + + pip install tahoe-lafs + +3. Verify installation by checking for the version:: + + tahoe --version + +If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. + +You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. diff --git a/docs/Makefile b/docs/Makefile index ed9e59186..3d7b51f7f 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -214,3 +214,7 @@ pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: livehtml +livehtml: + sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/OS-X.rst b/docs/OS-X.rst deleted file mode 100644 index 29ac0474f..000000000 --- a/docs/OS-X.rst +++ /dev/null @@ -1,23 +0,0 @@ -============== -OS-X Packaging -============== - -Pre-built Tahoe-LAFS ".pkg" installers for OS-X are generated with each -source-code commit. These installers offer an easy way to get Tahoe and all -its dependencies installed on your Mac. They do not yet provide a -double-clickable application: after installation, you will have a "tahoe" -command-line tool, which you can use from a shell (a Terminal window) just as -if you'd installed from source. - -Installers are available from this directory: - - https://tahoe-lafs.org/source/tahoe-lafs/tarballs/OS-X-packages/ - -Download the latest .pkg file to your computer and double-click on it. This -will install to /Applications/tahoe.app, however the app icon there is not -how you use Tahoe (launching it will get you a dialog box with a reminder to -use Terminal). ``/Applications/tahoe.app/bin/tahoe`` is the executable. The -next shell you start ought to have that directory in your $PATH (thanks to a -file in ``/etc/paths.d/``), unless your ``.profile`` overrides it. - -Tahoe-LAFS is also easy to install with pip, as described in the README. diff --git a/docs/README.txt b/docs/README.txt index 87c9583d9..b571a2077 100644 --- a/docs/README.txt +++ b/docs/README.txt @@ -1,7 +1,30 @@ +If you are reading Tahoe-LAFS documentation +------------------------------------------- -Note: http://tahoe-lafs.readthedocs.io/en/latest/ is the preferred place to -read this documentation (GitHub doesn't render cross-document links or -images). If you're reading this on https://github.com/tahoe-lafs/tahoe-lafs , -or from a checked-out source tree, then either run `tox -e docs` and open -_build/html/index.html in your browser, or view the pre-rendered trunk copy -at http://tahoe-lafs.readthedocs.io/en/latest/ +If you are reading Tahoe-LAFS documentation at a code hosting site or +from a checked-out source tree, the preferred place to view the docs +is http://tahoe-lafs.readthedocs.io/en/latest/. Code-hosting sites do +not render cross-document links or images correctly. + + +If you are writing Tahoe-LAFS documentation +------------------------------------------- + +To edit Tahoe-LAFS docs, you will need a checked-out source tree. You +can edit the `.rst` files in this directory using a text editor, and +then generate HTML output using Sphinx, a program that can produce its +output in HTML and other formats. + +Files with `.rst` extension use reStructuredText markup format, which +is the format Sphinx natively handles. To learn more about Sphinx, and +for a friendly primer on reStructuredText, please see Sphinx project's +documentation, available at: + +https://www.sphinx-doc.org/ + +If you have `tox` installed, you can run `tox -e docs` and then open +the resulting docs/_build/html/index.html in your web browser. + +Note that Sphinx can also process Python docstrings to generate API +documentation. Tahoe-LAFS currently does not use Sphinx for this +purpose. diff --git a/docs/_static/media/image2.png b/docs/_static/media/image2.png index d8704f359..90e6b9375 100644 Binary files a/docs/_static/media/image2.png and b/docs/_static/media/image2.png differ diff --git a/docs/about.rst b/docs/about-tahoe.rst similarity index 98% rename from docs/about.rst rename to docs/about-tahoe.rst index 120abb079..348e148dd 100644 --- a/docs/about.rst +++ b/docs/about-tahoe.rst @@ -127,7 +127,7 @@ For more technical detail, please see the `the doc page`_ on the Wiki. Get Started =========== -To use Tahoe-LAFS, please see :doc:`INSTALL`. +To use Tahoe-LAFS, please see :doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`. License ======= diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 0badede98..d501b7af5 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -514,10 +514,10 @@ Command Examples the pattern will be matched against any level of the directory tree; it's still impossible to specify absolute path exclusions. -``tahoe backup --exclude-from=/path/to/filename ~ work:backups`` +``tahoe backup --exclude-from-utf-8=/path/to/filename ~ work:backups`` - ``--exclude-from`` is similar to ``--exclude``, but reads exclusion - patterns from ``/path/to/filename``, one per line. + ``--exclude-from-utf-8`` is similar to ``--exclude``, but reads exclusion + patterns from a UTF-8-encoded ``/path/to/filename``, one per line. ``tahoe backup --exclude-vcs ~ work:backups`` diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst index ee6371812..9d4f1dcec 100644 --- a/docs/frontends/FTP-and-SFTP.rst +++ b/docs/frontends/FTP-and-SFTP.rst @@ -7,11 +7,10 @@ Tahoe-LAFS SFTP Frontend 1. `SFTP Background`_ 2. `Tahoe-LAFS Support`_ 3. `Creating an Account File`_ -4. `Running An Account Server (accounts.url)`_ -5. `Configuring SFTP Access`_ -6. `Dependencies`_ -7. `Immutable and Mutable Files`_ -8. `Known Issues`_ +4. `Configuring SFTP Access`_ +5. `Dependencies`_ +6. `Immutable and Mutable Files`_ +7. `Known Issues`_ SFTP Background @@ -78,33 +77,6 @@ start with "ssh-". Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in the next sections. -Running An Account Server (accounts.url) -======================================== - -The accounts.url directive allows access requests to be controlled by an -HTTP-based login service, useful for centralized deployments. This was used -by AllMyData to provide web-based file access, where the service used a -simple PHP script and database lookups to map an account email address and -password to a Tahoe-LAFS directory cap. The service will receive a -multipart/form-data POST, just like one created with a
and -fields, with three parameters: - -• action: "authenticate" (this is a static string) -• email: USERNAME (Tahoe-LAFS has no notion of email addresses, but the - authentication service uses them as account names, so the interface - presents this argument as "email" rather than "username"). -• passwd: PASSWORD - -It should return a single string that either contains a Tahoe-LAFS directory -cap (URI:DIR2:...), or "0" to indicate a login failure. - -Tahoe-LAFS recommends the service be secure, preferably localhost-only. This -makes it harder for attackers to brute force the password or use DNS -poisoning to cause the Tahoe-LAFS gateway to talk with the wrong server, -thereby revealing the usernames and passwords. - -Public key authentication is not supported when an account server is used. - Configuring SFTP Access ======================= diff --git a/docs/index.rst b/docs/index.rst index 6dad6c217..71c4145f5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,8 +10,11 @@ Contents: .. toctree:: :maxdepth: 2 - about - INSTALL + about-tahoe + Installation/install-tahoe + Installation/install-on-windows + Installation/install-on-linux + Installation/install-on-desert-island running magic-wormhole-invites configuration @@ -42,6 +45,7 @@ Contents: backupdb developer-guide + ticket-triage anonymity-configuration @@ -50,10 +54,7 @@ Contents: logging stats - desert-island debian - windows - OS-X build/build-pyOpenSSL specifications/index diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 1bdc774de..de0918b58 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -13,6 +13,102 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change (though this may be the first step towards simplifying it). +Glossary +-------- + +.. glossary:: + + `Foolscap `_ + an RPC/RMI (Remote Procedure Call / Remote Method Invocation) protocol for use with Twisted + + storage server + a Tahoe-LAFS process configured to offer storage and reachable over the network for store and retrieve operations + + introducer + a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers + + fURL + a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol + + lease + state associated with a share informing a storage server of the duration of storage desired by a client + + share + a single unit of client-provided arbitrary data to be stored by a storage server + (in practice, one of the outputs of applying ZFEC encoding to some ciphertext with some additional metadata attached) + + bucket + a group of one or more immutable shares held by a storage server and having a common storage index + + slot + a group of one or more mutable shares held by a storage server and having a common storage index + (sometimes "slot" is considered a synonym for "storage index of a slot") + + storage index + a short string which can address a slot or a bucket + (in practice, derived by hashing the encryption key associated with contents of that slot or bucket) + + write enabler + a short secret string which storage servers require to be presented before allowing mutation of any mutable share + + lease renew secret + a short secret string which storage servers required to be presented before allowing a particular lease to be renewed + +Motivation +---------- + +Foolscap +~~~~~~~~ + +Foolscap is a remote method invocation protocol with several distinctive features. +At its core it allows separate processes to refer each other's objects and methods using a capability-based model. +This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated. +Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways. + +Tahoe-LAFS avails itself of only a small portion of these features. +A Tahoe-LAFS storage server typically only exposes one object with a fixed set of methods to clients. +A Tahoe-LAFS introducer node does roughly the same. +Tahoe-LAFS exchanges simple data structures that have many common, standard serialized representations. + +In exchange for this slight use of Foolscap's sophisticated mechanisms, +Tahoe-LAFS pays a substantial price: + +* Foolscap is implemented only for Python. + Tahoe-LAFS is thus limited to being implemented only in Python. +* There is only one Python implementation of Foolscap. + The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. +* The Foolscap developer community is very small. + The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project. +* The extensible serialization system imposes substantial complexity compared to the simple data structures Tahoe-LAFS actually exchanges. + +HTTP +~~~~ + +HTTP is a request/response protocol that has become the lingua franca of the internet. +Combined with the principles of Representational State Transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. +HTTP itself provides only modest functionality in comparison to Foolscap. +However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. + +By adopting HTTP in place of Foolscap Tahoe-LAFS can realize the following concrete benefits: + +* Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available. + This change paves the way for new Tahoe-LAFS implementations using tools better suited for certain situations + (mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc). +* The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use. + This change reduces the barrier to entry for developers to contribute improvements to Tahoe-LAFS's network interactions. +* For any given language there is very likely an HTTP implementation with a large and active developer community. + Tahoe-LAFS can therefore benefit from the large effort being put into making better libraries for using HTTP. +* One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. + The alignment of this core feature with a core activity of Tahoe-LAFS of transferring bulk data means that a substantial barrier to improved Tahoe-LAFS runtime performance will be eliminated. + +TLS +~~~ + +The Foolscap-based protocol provides *some* of Tahoe-LAFS's confidentiality, integrity, and authentication properties by leveraging TLS. +An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties. +Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation +(rather than the standard "web" rules for validation). + Requirements ------------ @@ -234,6 +330,19 @@ Because of the simple types used throughout and the equivalence described in `RFC 7049`_ these examples should be representative regardless of which of these two encodings is chosen. +HTTP Design +~~~~~~~~~~~ + +The HTTP interface described here is informed by the ideas of REST +(Representational State Transfer). +For ``GET`` requests query parameters are preferred over values encoded in the request body. +For other requests query parameters are encoded into the message body. + +Many branches of the resource tree are conceived as homogenous containers: +one branch contains all of the share data; +another branch contains all of the lease data; +etc. + General ~~~~~~~ @@ -257,6 +366,71 @@ For example:: "application-version": "1.13.0" } +``PUT /v1/lease/:storage_index`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Create a new lease on the bucket addressed by ``storage_index``. +The details of the lease are encoded in the request body. +For example:: + + {"renew-secret": "abcd", "cancel-secret": "efgh"} + +If the ``renew-secret`` value matches an existing lease +then the expiration time of that lease will be changed to 31 days after the time of this operation. +If it does not match an existing lease +then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. + +In these cases the response is ``NO CONTENT`` with an empty body. + +It is possible that the storage server will have no shares for the given ``storage_index`` because: + +* no such shares have ever been uploaded. +* a previous lease expired and the storage server reclaimed the storage by deleting the shares. + +In these cases the server takes no action and returns ``NOT FOUND``. + + +Discussion +`````````` + +We considered an alternative where ``renew-secret`` and ``cancel-secret`` are placed in query arguments on the request path. +We chose to put these values into the request body to make the URL simpler. + +Several behaviors here are blindly copied from the Foolscap-based storage server protocol. + +* There is a cancel secret but there is no API to use it to cancel a lease (see ticket:3768). +* The lease period is hard-coded at 31 days. +* There are separate **add** and **renew** lease APIs (see ticket:3773). + +These are not necessarily ideal behaviors +but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. +It is expected that some or all of these behaviors may change in a future revision of the HTTP-based protocol. + +``POST /v1/lease/:storage_index`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +Renew an existing lease for all shares for the given storage index. +The details of the lease are encoded in the request body. +For example:: + + {"renew-secret": "abcd"} + +If there are no shares for the given ``storage_index`` +then ``NOT FOUND`` is returned. + +If there is no lease with a matching ``renew-secret`` value on the given storage index +then ``NOT FOUND`` is returned. +In this case, +if the storage index refers to mutable data +then the response also includes a list of nodeids where the lease can be renewed. +For example:: + + {"nodeids": ["aaa...", "bbb..."]} + +Othewise, +the matching lease's expiration time is changed to be 31 days from the time of this operation +and ``NO CONTENT`` is returned. + Immutable --------- @@ -268,6 +442,7 @@ Writing Initialize an immutable storage index with some buckets. The buckets may have share data written to them once. +A lease is also created for the shares. Details of the buckets to create are encoded in the request body. For example:: @@ -294,6 +469,15 @@ However, we decided this does not matter because: therefore no proxy servers can perform any extra logging. * Tahoe-LAFS itself does not currently log HTTP request URLs. +The response includes ``already-have`` and ``allocated`` for two reasons: + +* If an upload is interrupted and the client loses its local state that lets it know it already uploaded some shares + then this allows it to discover this fact (by inspecting ``already-have``) and only upload the missing shares (indicated by ``allocated``). + +* If an upload has completed a client may still choose to re-balance storage by moving shares between servers. + This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. + It could also just be that the client's preferred servers have changed. + ``PUT /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -448,6 +632,136 @@ Just like ``GET /v1/mutable/:storage_index``. Advise the server the data read from the indicated share was corrupt. Just like the immutable version. +Sample Interactions +------------------- + +Immutable Data +~~~~~~~~~~~~~~ + +1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: + + POST /v1/immutable/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh", "cancel-secret": "ijkl", + "share-numbers": [1, 7], "allocated-size": 48} + + 200 OK + {"already-have": [1], "allocated": [7]} + +#. Upload the content for immutable share ``7``:: + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 0-15/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 16-31/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 32-47/48 + + + 201 CREATED + +#. Download the content of the previously uploaded immutable share ``7``:: + + GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7&offset=0&size=48 + + 200 OK + + +#. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: + + POST /v1/lease/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh"} + + 204 NO CONTENT + +Mutable Data +~~~~~~~~~~~~ + +1. Create mutable share number ``3`` with ``10`` bytes of data in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": 1, + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "xxxxxxxxxx" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": , + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "yyyyyyyyyy" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Download the contents of share number ``3``:: + + GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + + +#. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/lease/BBBBBBBBBBBBBBBB + {"renew-secret": "efgh"} + + 204 NO CONTENT + .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4 diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 75ab74bb1..a5761c1c7 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -59,6 +59,10 @@ Create Branch and Apply Updates - summarize major changes - commit it +- update "nix/tahoe-lafs.nix" + + - change the value given for `version` from `OLD.post1` to `NEW.post1` + - update "CREDITS" - are there any new contributors in this release? @@ -66,7 +70,7 @@ Create Branch and Apply Updates - commit it - update "docs/known_issues.rst" if appropriate -- update "docs/INSTALL.rst" references to the new release +- update "docs/Installation/install-tahoe.rst" references to the new release - Push the branch to github - Create a (draft) PR; this should trigger CI (note that github doesn't let you create a PR without some changes on the branch so @@ -189,11 +193,16 @@ is appropriate. Once a release-candidate has marinated for some time then it can be made into a the actual release. -XXX Write this section when doing 1.15.0 actual release - -(In general, this means dropping the "rcX" part of the release and the -tag, uploading those artifacts, uploading to PyPI, ... ) +The actual release follows the same steps as above, with some differences: +- there is no "-rcX" on the end of release names +- the release is uploaded to PyPI (using Twine) +- the version is tagged in Git (ideally using "the tahoe release key" + but can be done with any of the authorized core developers' personal + key) +- the release-candidate branches must be merged back to master after + the release is official (e.g. causing newsfragments to be deleted on + master, etc) Announcing the Release diff --git a/docs/running.rst b/docs/running.rst index 82b0443f9..a53f5d9e2 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -10,7 +10,7 @@ Introduction This is how to run a Tahoe-LAFS client or a complete Tahoe-LAFS grid. First you have to install the Tahoe-LAFS software, as documented in -:doc:`INSTALL`. +:doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`. The ``tahoe`` program in your virtualenv's ``bin`` directory is used to create, start, and stop nodes. Each node lives in a separate base @@ -235,7 +235,7 @@ Socialize ========= You can chat with other users of and hackers of this software on the -#tahoe-lafs IRC channel at ``irc.freenode.net``, or on the `tahoe-dev mailing +#tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing list`_. .. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev diff --git a/docs/ticket-triage.rst b/docs/ticket-triage.rst new file mode 100644 index 000000000..b92232507 --- /dev/null +++ b/docs/ticket-triage.rst @@ -0,0 +1,27 @@ +============= +Ticket Triage +============= + +Ticket triage is a weekly, informal ritual that is meant to solve the problem of +tickets getting opened and then forgotten about. It is simple and keeps project +momentum going and prevents ticket cruft. + +It fosters conversation around project tasks and philosophies as they relate to +milestones. + +Process +------- +- The role of Ticket Triager rotates regularly-ish, and is assigned ad hoc +- The Triager needs a ``Trac`` account +- The Triager looks at all the tickets that have been created in the last week (or month, etc.) + - They can use a custom query or do this as the week progresses + - BONUS ROUND: Dig up a stale ticket from the past +- Assign each ticket to a milestone on the Roadmap +- The following situations merit discussion: + - A ticket doesn't have an appropriate milestone and we should create one + - A ticket, in vanishingly rare circumstances, should be deleted + - The ticket is spam + - The ticket contains sensitive information and harm will come to one or more people if it continues to be distributed + - A ticket could be assigned to multiple milestones + - There is another question about a ticket +- These tickets will be brought as necessary to one of our meetings (currently Tuesdays) for discussion diff --git a/docs/windows.rst b/docs/windows.rst deleted file mode 100644 index 1f69ac743..000000000 --- a/docs/windows.rst +++ /dev/null @@ -1,83 +0,0 @@ -Building Tahoe-LAFS on Windows -============================== - -You'll need ``python``, ``pip``, and ``virtualenv``. But you won't need a -compiler. - -Preliminaries -------------- - -1: Install Python-2.7.11 . Use the "Windows x86-64 MSI installer" at -https://www.python.org/downloads/release/python-2711/ - -2: That should install ``pip``, but if it doesn't, look at -https://pip.pypa.io/en/stable/installing/ for installation instructions. - -3: Install ``virtualenv`` with -https://virtualenv.pypa.io/en/latest/installation.html - -Installation ------------- - -1: Start a CLI shell (e.g. PowerShell) - -2: Create a new virtualenv. Everything specific to Tahoe will go into this. -You can use whatever name you like for the virtualenv, but example uses -"venv":: - - PS C:\Users\me> virtualenv venv - New python executable in C:\Users\me\venv\Scripts\python.exe - Installing setuptools, pip, wheel...done. - > - -3: Use the virtualenv's ``pip`` to install the latest release of Tahoe-LAFS -into this virtualenv:: - - PS C:\Users\me> venv\Scripts\pip install tahoe-lafs - Collecting tahoe-lafs - ... - Installing collected packages: ... - Successfully installed ... - > - -4: Verify that Tahoe was installed correctly by running ``tahoe --version``, -using the ``tahoe`` from the virtualenv's Scripts directory:: - - PS C:\Users\me> venv\Scripts\tahoe --version - tahoe-lafs: 1.11 - foolscap: ... - -Running Tahoe-LAFS ------------------- - -The rest of the documentation assumes you can run the ``tahoe`` executable -just as you did in step 4 above. If you want to type just ``tahoe`` instead -of ``venv\Scripts\tahoe``, you can either "`activate`_" the virtualenv (by -running ``venv\Scripts\activate``, or you can add the Scripts directory to -your ``%PATH%`` environment variable. - -Now use the docs in :doc:`running` to learn how to configure your first -Tahoe node. - -.. _activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script - -Installing A Different Version ------------------------------- - -The ``pip install tahoe-lafs`` command above will install the latest release -(from PyPI). If instead, you want to install from a git checkout, then run -the following command (using pip from the virtualenv, from the root of your -git checkout):: - - $ venv\Scripts\pip install . - -If you're planning to hack on the source code, you might want to add -``--editable`` so you won't have to re-install each time you make a change. - -Dependencies ------------- - -Tahoe-LAFS depends upon several packages that use compiled C code (such as zfec). -This code must be built separately for each platform (Windows, OS-X, and different flavors of Linux). -Fortunately, this is now done by upstream packages for most platforms. -The result is that a C compiler is usually not required to install Tahoe-LAFS. diff --git a/integration/conftest.py b/integration/conftest.py index 533cbdb67..39ff3b42b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys import shutil from time import sleep @@ -28,7 +38,7 @@ from twisted.internet.error import ( import pytest import pytest_twisted -from util import ( +from .util import ( _CollectOutputProtocol, _MagicTextProtocol, _DumpOutputProtocol, diff --git a/integration/test_aaa_aardvark.py b/integration/test_aaa_aardvark.py index 4a2ef71a6..28ac4c412 100644 --- a/integration/test_aaa_aardvark.py +++ b/integration/test_aaa_aardvark.py @@ -5,6 +5,15 @@ # You can safely skip any of these tests, it'll just appear to "take # longer" to start the first test as the fixtures get built +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + def test_create_flogger(flog_gatherer): print("Created flog_gatherer") diff --git a/integration/test_get_put.py b/integration/test_get_put.py new file mode 100644 index 000000000..bbdc363ea --- /dev/null +++ b/integration/test_get_put.py @@ -0,0 +1,64 @@ +""" +Integration tests for getting and putting files, including reading from stdin +and stdout. +""" + +from subprocess import Popen, PIPE + +import pytest + +from .util import run_in_thread, cli + +DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" +try: + DATA.decode("utf-8") +except UnicodeDecodeError: + pass # great, what we want +else: + raise ValueError("BUG, the DATA string was decoded from UTF-8") + + +@pytest.fixture(scope="session") +def get_put_alias(alice): + cli(alice, "create-alias", "getput") + + +def read_bytes(path): + with open(path, "rb") as f: + return f.read() + + +@run_in_thread +def test_put_from_stdin(alice, get_put_alias, tmpdir): + """ + It's possible to upload a file via `tahoe put`'s STDIN, and then download + it to a file. + """ + tempfile = str(tmpdir.join("file")) + p = Popen( + ["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"], + stdin=PIPE + ) + p.stdin.write(DATA) + p.stdin.close() + assert p.wait() == 0 + + cli(alice, "get", "getput:fromstdin", tempfile) + assert read_bytes(tempfile) == DATA + + +def test_get_to_stdout(alice, get_put_alias, tmpdir): + """ + It's possible to upload a file, and then download it to stdout. + """ + tempfile = tmpdir.join("file") + with tempfile.open("wb") as f: + f.write(DATA) + cli(alice, "put", str(tempfile), "getput:tostdout") + + p = Popen( + ["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"], + stdout=PIPE + ) + assert p.stdout.read() == DATA + assert p.wait() == 0 diff --git a/integration/test_i2p.py b/integration/test_i2p.py new file mode 100644 index 000000000..f0b06f1e2 --- /dev/null +++ b/integration/test_i2p.py @@ -0,0 +1,244 @@ +""" +Integration tests for I2P support. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import sys +from os.path import join, exists +from os import mkdir +from time import sleep + +if PY2: + def which(path): + # This will result in skipping I2P tests on Python 2. Oh well. + return None +else: + from shutil import which + +from eliot import log_call + +import pytest +import pytest_twisted + +from . import util + +from twisted.python.filepath import ( + FilePath, +) +from twisted.internet.error import ProcessExitedAlready + +from allmydata.test.common import ( + write_introducer, +) + +if which("docker") is None: + pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) +# Docker on Windows machines sometimes expects Windows-y Docker images, so just +# don't bother. +if sys.platform.startswith('win'): + pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) + + +@pytest.fixture +def i2p_network(reactor, temp_dir, request): + """Fixture to start up local i2pd.""" + proto = util._MagicTextProtocol("ephemeral keys") + reactor.spawnProcess( + proto, + which("docker"), + ( + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + # Bad URL for reseeds, so it can't talk to other routers. + "--reseed.urls", "http://localhost:1/", + ), + ) + + def cleanup(): + try: + proto.transport.signalProcess("KILL") + util.block_with_timeout(proto.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + util.block_with_timeout(proto.magic_seen, reactor, timeout=30) + + +@pytest.fixture +@log_call( + action_type=u"integration:i2p:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def i2p_introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer_i2p +web.port = 4561 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer_i2p') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = util._ProcessExitedProtocol() + util._tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=i2p', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old + # "start" command. + protocol = util._MagicTextProtocol('introducer running') + transport = util._tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def cleanup(): + try: + transport.signalProcess('TERM') + util.block_with_timeout(protocol.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + pytest_twisted.blockon(protocol.magic_seen) + return transport + + +@pytest.fixture +def i2p_introducer_furl(i2p_introducer, temp_dir): + furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl + + +@pytest_twisted.inlineCallbacks +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): + yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + # ensure both nodes are connected to "a grid" by uploading + # something via carol, and retrieve it using dave. + gold_path = join(temp_dir, "gold") + with open(gold_path, "w") as f: + f.write( + "The object-capability model is a computer security model. A " + "capability describes a transferable right to perform one (or " + "more) operations on a given object." + ) + # XXX could use treq or similar to POST these to their respective + # WUIs instead ... + + proto = util._CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'carol_i2p'), + 'put', gold_path, + ) + ) + yield proto.done + cap = proto.output.getvalue().strip().split()[-1] + print("TEH CAP!", cap) + + proto = util._CollectOutputProtocol(capture_stderr=False) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'dave_i2p'), + 'get', cap, + ) + ) + yield proto.done + + dave_got = proto.output.getvalue().strip() + assert dave_got == open(gold_path, 'rb').read().strip() + + +@pytest_twisted.inlineCallbacks +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): + node_dir = FilePath(temp_dir).child(name) + web_port = "tcp:{}:interface=localhost".format(control_port + 2000) + + print("creating", node_dir.path) + node_dir.makedirs() + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--listen', 'i2p', + node_dir.path, + ) + ) + yield proto.done + + + # Which services should this client connect to? + write_introducer(node_dir, "default", introducer_furl) + with node_dir.child('tahoe.cfg').open('w') as f: + node_config = ''' +[node] +nickname = %(name)s +web.port = %(web_port)s +web.static = public_html +log_gatherer.furl = %(log_furl)s + +[i2p] +enabled = true + +[client] +shares.needed = 1 +shares.happy = 1 +shares.total = 2 + +''' % { + 'name': name, + 'web_port': web_port, + 'log_furl': flog_gatherer, +} + node_config = node_config.encode("utf-8") + f.write(node_config) + + print("running") + yield util._run_node(reactor, node_dir.path, request, None) + print("okay, launched") diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py index 97392bf00..b9de0c075 100644 --- a/integration/test_servers_of_happiness.py +++ b/integration/test_servers_of_happiness.py @@ -1,9 +1,21 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join from twisted.internet.error import ProcessTerminated -import util +from . import util import pytest_twisted @@ -30,7 +42,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto proto, sys.executable, [ - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', node_dir, 'put', __file__, ] @@ -42,4 +54,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto assert isinstance(e, ProcessTerminated) output = proto.output.getvalue() - assert "shares could be placed on only" in output + assert b"shares could be placed on only" in output diff --git a/integration/test_streaming_logs.py b/integration/test_streaming_logs.py index 52c813f9b..036d30715 100644 --- a/integration/test_streaming_logs.py +++ b/integration/test_streaming_logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -5,12 +8,18 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_text + import json from os.path import ( join, ) -from urlparse import ( +from urllib.parse import ( urlsplit, ) @@ -68,7 +77,7 @@ def _connect_client(reactor, api_auth_token, ws_url): factory = WebSocketClientFactory( url=ws_url, headers={ - "Authorization": "{} {}".format(SCHEME, api_auth_token), + "Authorization": "{} {}".format(str(SCHEME, "ascii"), api_auth_token), } ) factory.protocol = _StreamingLogClientProtocol @@ -127,7 +136,7 @@ def _test_streaming_logs(reactor, temp_dir, alice): node_url = cfg.get_config_from_file("node.url") api_auth_token = cfg.get_private_config("api_auth_token") - ws_url = node_url.replace("http://", "ws://") + ws_url = ensure_text(node_url).replace("http://", "ws://") log_url = ws_url + "private/logs/v1" print("Connecting to {}".format(log_url)) diff --git a/integration/test_tor.py b/integration/test_tor.py index dcbfb1151..15d888e36 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -1,12 +1,22 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys from os.path import join import pytest import pytest_twisted -import util +from . import util from twisted.python.filepath import ( FilePath, @@ -46,7 +56,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'carol'), 'put', gold_path, ) @@ -55,12 +65,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne cap = proto.output.getvalue().strip().split()[-1] print("TEH CAP!", cap) - proto = util._CollectOutputProtocol() + proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', '-d', join(temp_dir, 'dave'), 'get', cap, ) @@ -68,7 +78,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne yield proto.done dave_got = proto.output.getvalue().strip() - assert dave_got == open(gold_path, 'r').read().strip() + assert dave_got == open(gold_path, 'rb').read().strip() @pytest_twisted.inlineCallbacks @@ -84,7 +94,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ proto, sys.executable, ( - sys.executable, '-m', 'allmydata.scripts.runner', + sys.executable, '-b', '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, '--introducer', introducer_furl, @@ -100,7 +110,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) with node_dir.child('tahoe.cfg').open('w') as f: - f.write(''' + node_config = ''' [node] nickname = %(name)s web.port = %(web_port)s @@ -125,7 +135,9 @@ shares.total = 2 'log_furl': flog_gatherer, 'control_port': control_port, 'local_port': control_port + 1000, -}) +} + node_config = node_config.encode("utf-8") + f.write(node_config) print("running") yield util._run_node(reactor, node_dir.path, request, None) diff --git a/integration/test_web.py b/integration/test_web.py index aab11412f..22f08da82 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -7,15 +7,26 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did at the time of testing' -- not necessarily a cohesive idea of what the WebAPI *should* do in every situation. It's not clear the latter exists anywhere, however. + +Ported to Python 3. """ +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time -import json -import urllib2 +from urllib.parse import unquote as url_unquote, quote as url_quote import allmydata.uri +from allmydata.util import jsonbytes as json -import util +from . import util import requests import html5lib @@ -64,7 +75,7 @@ def test_upload_download(alice): u"filename": u"boom", } ) - assert data == FILE_CONTENTS + assert str(data, "utf-8") == FILE_CONTENTS def test_put(alice): @@ -95,7 +106,7 @@ def test_helper_status(storage_nodes): resp = requests.get(url) assert resp.status_code >= 200 and resp.status_code < 300 dom = BeautifulSoup(resp.content, "html5lib") - assert unicode(dom.h1.string) == u"Helper Status" + assert str(dom.h1.string) == u"Helper Status" def test_deep_stats(alice): @@ -115,10 +126,10 @@ def test_deep_stats(alice): # when creating a directory, we'll be re-directed to a URL # containing our writecap.. - uri = urllib2.unquote(resp.url) + uri = url_unquote(resp.url) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') - dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap))) + dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap))) # POST a file into this directory FILE_CONTENTS = u"a file in a directory" @@ -145,7 +156,7 @@ def test_deep_stats(alice): k, data = d assert k == u"dirnode" assert len(data['children']) == 1 - k, child = data['children'].values()[0] + k, child = list(data['children'].values())[0] assert k == u"filenode" assert child['size'] == len(FILE_CONTENTS) @@ -196,11 +207,11 @@ def test_status(alice): print("Uploaded data, cap={}".format(cap)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))), ) print("Downloaded {} bytes of data".format(len(resp.content))) - assert resp.content == FILE_CONTENTS + assert str(resp.content, "ascii") == FILE_CONTENTS resp = requests.get( util.node_url(alice.node_dir, "status"), @@ -219,12 +230,12 @@ def test_status(alice): continue resp = requests.get(util.node_url(alice.node_dir, href)) if href.startswith(u"/status/up"): - assert "File Upload Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Upload Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_upload = True elif href.startswith(u"/status/down"): - assert "File Download Status" in resp.content - if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: + assert b"File Download Status" in resp.content + if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content: found_download = True # download the specialized event information @@ -297,7 +308,7 @@ def test_directory_deep_check(alice): print("Uploaded data1, cap={}".format(cap1)) resp = requests.get( - util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))), + util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))), params={u"t": u"info"}, ) @@ -398,9 +409,9 @@ def test_directory_deep_check(alice): for _ in range(5): resp = requests.get(deepcheck_uri) dom = BeautifulSoup(resp.content, "html5lib") - if dom.h1 and u'Results' in unicode(dom.h1.string): + if dom.h1 and u'Results' in str(dom.h1.string): break - if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string): + if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string): dom = None time.sleep(1) assert dom is not None, "Operation never completed" @@ -438,7 +449,7 @@ def test_introducer_info(introducer): resp = requests.get( util.node_url(introducer.node_dir, u""), ) - assert "Introducer" in resp.content + assert b"Introducer" in resp.content resp = requests.get( util.node_url(introducer.node_dir, u""), @@ -511,6 +522,6 @@ def test_mkdir_with_children(alice): params={u"t": "mkdir-with-children"}, data=json.dumps(meta), ) - assert resp.startswith("URI:DIR2") + assert resp.startswith(b"URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI) diff --git a/integration/util.py b/integration/util.py index 256fd68c1..7c7a1efd2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,9 +1,21 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import sys import time import json from os import mkdir, environ from os.path import exists, join -from six.moves import StringIO +from io import StringIO, BytesIO from functools import partial from subprocess import check_output @@ -55,9 +67,10 @@ class _CollectOutputProtocol(ProcessProtocol): self.output, and callback's on done with all of it after the process exits (for any reason). """ - def __init__(self): + def __init__(self, capture_stderr=True): self.done = Deferred() - self.output = StringIO() + self.output = BytesIO() + self.capture_stderr = capture_stderr def processEnded(self, reason): if not self.done.called: @@ -71,8 +84,9 @@ class _CollectOutputProtocol(ProcessProtocol): self.output.write(data) def errReceived(self, data): - print("ERR: {}".format(data)) - self.output.write(data) + print("ERR: {!r}".format(data)) + if self.capture_stderr: + self.output.write(data) class _DumpOutputProtocol(ProcessProtocol): @@ -92,9 +106,11 @@ class _DumpOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): + data = str(data, sys.stdout.encoding) self._out.write(data) def errReceived(self, data): + data = str(data, sys.stdout.encoding) self._out.write(data) @@ -114,6 +130,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): + data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): @@ -121,6 +138,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): + data = str(data, sys.stderr.encoding) sys.stdout.write(data) @@ -152,9 +170,9 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args): `--coverage` option if the `request` indicates we should. """ if request.config.getoption('coverage'): - args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] + args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage'] else: - args = [sys.executable, '-m', 'allmydata.scripts.runner'] + args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner'] args += other_args return reactor.spawnProcess( proto, @@ -261,9 +279,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--hostname', 'localhost', '--listen', 'tcp', '--webport', web_port, - '--shares-needed', unicode(needed), - '--shares-happy', unicode(happy), - '--shares-total', unicode(total), + '--shares-needed', str(needed), + '--shares-happy', str(happy), + '--shares-total', str(total), '--helper', ] if not storage: @@ -280,7 +298,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam config, u'node', u'log_gatherer.furl', - flog_gatherer.decode("utf-8"), + flog_gatherer, ) write_config(FilePath(config_path), config) created_d.addCallback(created) @@ -526,7 +544,8 @@ def generate_ssh_key(path): key = RSAKey.generate(2048) key.write_private_key_file(path) with open(path + ".pub", "wb") as f: - f.write(b"%s %s" % (key.get_name(), key.get_base64())) + s = "%s %s" % (key.get_name(), key.get_base64()) + f.write(s.encode("ascii")) def run_in_thread(f): diff --git a/src/allmydata/test/check_grid.py b/misc/checkers/check_grid.py similarity index 100% rename from src/allmydata/test/check_grid.py rename to misc/checkers/check_grid.py diff --git a/src/allmydata/test/check_load.py b/misc/checkers/check_load.py similarity index 100% rename from src/allmydata/test/check_load.py rename to misc/checkers/check_load.py diff --git a/src/allmydata/test/check_memory.py b/misc/checkers/check_memory.py similarity index 100% rename from src/allmydata/test/check_memory.py rename to misc/checkers/check_memory.py diff --git a/src/allmydata/test/check_speed.py b/misc/checkers/check_speed.py similarity index 100% rename from src/allmydata/test/check_speed.py rename to misc/checkers/check_speed.py diff --git a/misc/coding_tools/check-debugging.py b/misc/coding_tools/check-debugging.py index f2ba6528e..b920f5634 100755 --- a/misc/coding_tools/check-debugging.py +++ b/misc/coding_tools/check-debugging.py @@ -1,13 +1,18 @@ #! /usr/bin/python -# ./check-debugging.py src +""" +Checks for defer.setDebugging(). + +Runs on Python 3. + +Usage: ./check-debugging.py src +""" from __future__ import print_function import sys, re, os ok = True -umids = {} for starting_point in sys.argv[1:]: for root, dirs, files in os.walk(starting_point): diff --git a/misc/coding_tools/check-miscaptures.py b/misc/coding_tools/check-miscaptures.py deleted file mode 100644 index 81e76f891..000000000 --- a/misc/coding_tools/check-miscaptures.py +++ /dev/null @@ -1,186 +0,0 @@ -#! /usr/bin/python - -from __future__ import print_function - -import os, sys, compiler -from compiler.ast import Node, For, While, ListComp, AssName, Name, Lambda, Function - - -def check_source(source): - return check_thing(compiler.parse, source) - -def check_file(path): - return check_thing(compiler.parseFile, path) - -def check_thing(parser, thing): - try: - ast = parser(thing) - except SyntaxError as e: - return e - else: - results = [] - check_ast(ast, results) - return results - -def check_ast(ast, results): - """Check a node outside a loop.""" - if isinstance(ast, (For, While, ListComp)): - check_loop(ast, results) - else: - for child in ast.getChildNodes(): - if isinstance(ast, Node): - check_ast(child, results) - -def check_loop(ast, results): - """Check a particular outer loop.""" - - # List comprehensions have a poorly designed AST of the form - # ListComp(exprNode, [ListCompFor(...), ...]), in which the - # result expression is outside the ListCompFor node even though - # it is logically inside the loop(s). - # There may be multiple ListCompFor nodes (in cases such as - # [lambda: (a,b) for a in ... for b in ...] - # ), and that case they are not nested in the AST. But these - # warts (nonobviously) happen not to matter for our analysis. - - assigned = {} # maps name to lineno of topmost assignment - nested = set() - collect_assigned_and_nested(ast, assigned, nested) - - # For each nested function... - for funcnode in nested: - # Check for captured variables in this function. - captured = set() - collect_captured(funcnode, assigned, captured, False) - for name in captured: - # We want to report the outermost capturing function - # (since that is where the workaround will need to be - # added), and the topmost assignment to the variable. - # Just one report per capturing function per variable - # will do. - results.append(make_result(funcnode, name, assigned[name])) - - # Check each node in the function body in case it - # contains another 'for' loop. - childnodes = funcnode.getChildNodes()[len(funcnode.defaults):] - for child in childnodes: - check_ast(child, results) - -def collect_assigned_and_nested(ast, assigned, nested): - """ - Collect the names assigned in this loop, not including names - assigned in nested functions. Also collect the nodes of functions - that are nested one level deep. - """ - if isinstance(ast, AssName): - if ast.name not in assigned or assigned[ast.name] > ast.lineno: - assigned[ast.name] = ast.lineno - else: - childnodes = ast.getChildNodes() - if isinstance(ast, (Lambda, Function)): - nested.add(ast) - - # The default argument expressions are "outside" the - # function, even though they are children of the - # Lambda or Function node. - childnodes = childnodes[:len(ast.defaults)] - - for child in childnodes: - if isinstance(ast, Node): - collect_assigned_and_nested(child, assigned, nested) - -def collect_captured(ast, assigned, captured, in_function_yet): - """Collect any captured variables that are also in assigned.""" - if isinstance(ast, Name): - if ast.name in assigned: - captured.add(ast.name) - else: - childnodes = ast.getChildNodes() - if isinstance(ast, (Lambda, Function)): - # Formal parameters of the function are excluded from - # captures we care about in subnodes of the function body. - new_assigned = assigned.copy() - remove_argnames(ast.argnames, new_assigned) - - if len(new_assigned) > 0: - for child in childnodes[len(ast.defaults):]: - collect_captured(child, new_assigned, captured, True) - - # The default argument expressions are "outside" *this* - # function, even though they are children of the Lambda or - # Function node. - if not in_function_yet: - return - childnodes = childnodes[:len(ast.defaults)] - - for child in childnodes: - if isinstance(ast, Node): - collect_captured(child, assigned, captured, True) - - -def remove_argnames(names, fromset): - for element in names: - if element in fromset: - del fromset[element] - elif isinstance(element, (tuple, list)): - remove_argnames(element, fromset) - - -def make_result(funcnode, var_name, var_lineno): - if hasattr(funcnode, 'name'): - func_name = 'function %r' % (funcnode.name,) - else: - func_name = '' - return (funcnode.lineno, func_name, var_name, var_lineno) - -def report(out, path, results): - for r in results: - print(path + (":%r %s captures %r assigned at line %d" % r), file=out) - -def check(sources, out): - class Counts(object): - n = 0 - processed_files = 0 - suspect_files = 0 - error_files = 0 - counts = Counts() - - def _process(path): - results = check_file(path) - if isinstance(results, SyntaxError): - print(path + (" NOT ANALYSED due to syntax error: %s" % results), file=out) - counts.error_files += 1 - else: - report(out, path, results) - counts.n += len(results) - counts.processed_files += 1 - if len(results) > 0: - counts.suspect_files += 1 - - for source in sources: - print("Checking %s..." % (source,), file=out) - if os.path.isfile(source): - _process(source) - else: - for (dirpath, dirnames, filenames) in os.walk(source): - for fn in filenames: - (basename, ext) = os.path.splitext(fn) - if ext == '.py': - _process(os.path.join(dirpath, fn)) - - print("%d suspiciously captured variables in %d out of %d file(s)." - % (counts.n, counts.suspect_files, counts.processed_files), file=out) - if counts.error_files > 0: - print("%d file(s) not processed due to syntax errors." - % (counts.error_files,), file=out) - return counts.n - - -sources = ['src'] -if len(sys.argv) > 1: - sources = sys.argv[1:] -if check(sources, sys.stderr) > 0: - sys.exit(1) - - -# TODO: self-tests diff --git a/misc/coding_tools/check-umids.py b/misc/coding_tools/check-umids.py index c06b795fe..345610f3e 100644 --- a/misc/coding_tools/check-umids.py +++ b/misc/coding_tools/check-umids.py @@ -1,4 +1,10 @@ -#! /usr/bin/python +#! /usr/bin/python3 + +""" +Ensure UMIDS are unique. + +This runs on Python 3. +""" # ./check-umids.py src diff --git a/misc/coding_tools/make-canary-files.py b/misc/coding_tools/make-canary-files.py index fa813f047..89f274b38 100644 --- a/misc/coding_tools/make-canary-files.py +++ b/misc/coding_tools/make-canary-files.py @@ -52,6 +52,8 @@ system where Tahoe is installed, or in a source tree with setup.py like this: setup.py run_with_pythonpath -p -c 'misc/make-canary-files.py ARGS..' """ +from past.builtins import cmp + import os, hashlib from twisted.python import usage from allmydata.immutable import upload diff --git a/misc/operations_helpers/cpu-watcher.tac b/misc/operations_helpers/cpu-watcher.tac index 795b9c444..c50b51c61 100644 --- a/misc/operations_helpers/cpu-watcher.tac +++ b/misc/operations_helpers/cpu-watcher.tac @@ -201,7 +201,9 @@ class CPUWatcher(service.MultiService, resource.Resource, Referenceable): log.msg("error reading process %s (%s), ignoring" % (pid, name)) log.err() try: - pickle.dump(self.history, open("history.pickle.tmp", "wb")) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(self.history, open("history.pickle.tmp", "wb"), protocol=2) os.rename("history.pickle.tmp", "history.pickle") except: pass diff --git a/misc/operations_helpers/provisioning/provisioning.py b/misc/operations_helpers/provisioning/provisioning.py index d6dfc4cd7..d2fa44951 100644 --- a/misc/operations_helpers/provisioning/provisioning.py +++ b/misc/operations_helpers/provisioning/provisioning.py @@ -18,7 +18,7 @@ def factorial(n): factorial(n) with n<0 is -factorial(abs(n)) """ result = 1 - for i in xrange(1, abs(n)+1): + for i in range(1, abs(n)+1): result *= i assert n >= 0 return result @@ -30,7 +30,7 @@ def binomial(n, k): # calculate n!/k! as one product, avoiding factors that # just get canceled P = k+1 - for i in xrange(k+2, n+1): + for i in range(k+2, n+1): P *= i # if you are paranoid: # C, rem = divmod(P, factorial(n-k)) diff --git a/misc/simulators/hashbasedsig.py b/misc/simulators/hashbasedsig.py index dc141744f..dbb9ca504 100644 --- a/misc/simulators/hashbasedsig.py +++ b/misc/simulators/hashbasedsig.py @@ -79,7 +79,7 @@ def make_candidate(B, K, K1, K2, q, T, T_min, L_hash, lg_N, sig_bytes, c_sign, c # Winternitz with B < 4 is never optimal. For example, going from B=4 to B=2 halves the # chain depth, but that is cancelled out by doubling (roughly) the number of digits. -range_B = xrange(4, 33) +range_B = range(4, 33) M = pow(2, lg_M) @@ -100,7 +100,7 @@ def calculate(K, K1, K2, q_max, L_hash, trees): T_min = ceil_div(lg_M - lg_K1, lg_K) last_q = None - for T in xrange(T_min, T_min+21): + for T in range(T_min, T_min+21): # lg(total number of leaf private keys) lg_S = lg_K1 + lg_K*T lg_N = lg_S + lg_K2 @@ -137,14 +137,14 @@ def calculate(K, K1, K2, q_max, L_hash, trees): # We approximate lg(M-x) as lg(M) lg_px_step = lg_M + lg_p - lg_1_p - for x in xrange(1, j): + for x in range(1, j): lg_px[x] = lg_px[x-1] - lg(x) + lg_px_step q = None # Find the minimum acceptable value of q. - for q_cand in xrange(1, q_max+1): + for q_cand in range(1, q_max+1): lg_q = lg(q_cand) - lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in xrange(1, j)] + lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in range(1, j)] if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash: #print("K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f" # % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3)) @@ -246,13 +246,13 @@ def search(): K_max = 50 c2 = compressions(2*L_hash) c3 = compressions(3*L_hash) - for dau in xrange(0, 10): + for dau in range(0, 10): a = pow(2, dau) - for tri in xrange(0, ceil_log(30-dau, 3)): + for tri in range(0, ceil_log(30-dau, 3)): x = int(a*pow(3, tri)) h = dau + 2*tri c_x = int(sum_powers(2, dau)*c2 + a*sum_powers(3, tri)*c3) - for y in xrange(1, x+1): + for y in range(1, x+1): if tri > 0: # If the bottom level has arity 3, then for every 2 nodes by which the tree is # imperfect, we can save c3 compressions by pruning 3 leaves back to their parent. @@ -267,16 +267,16 @@ def search(): if y not in trees or (h, c_y, (dau, tri)) < trees[y]: trees[y] = (h, c_y, (dau, tri)) - #for x in xrange(1, K_max+1): + #for x in range(1, K_max+1): # print(x, trees[x]) candidates = [] progress = 0 fuzz = 0 complete = (K_max-1)*(2200-200)/100 - for K in xrange(2, K_max+1): - for K2 in xrange(200, 2200, 100): - for K1 in xrange(max(2, K-fuzz), min(K_max, K+fuzz)+1): + for K in range(2, K_max+1): + for K2 in range(200, 2200, 100): + for K1 in range(max(2, K-fuzz), min(K_max, K+fuzz)+1): candidates += calculate(K, K1, K2, q_max, L_hash, trees) progress += 1 print("searching: %3d %% \r" % (100.0 * progress / complete,), end=' ', file=stderr) @@ -285,7 +285,7 @@ def search(): step = 2.0 bins = {} limit = floor_div(limit_cost, step) - for bin in xrange(0, limit+2): + for bin in range(0, limit+2): bins[bin] = [] for c in candidates: @@ -296,7 +296,7 @@ def search(): # For each in a range of signing times, find the best candidate. best = [] - for bin in xrange(0, limit): + for bin in range(0, limit): candidates = bins[bin] + bins[bin+1] + bins[bin+2] if len(candidates) > 0: best += [min(candidates, key=lambda c: c['sig_bytes'])] diff --git a/misc/simulators/simulate_load.py b/misc/simulators/simulate_load.py index 945d96990..ed80ab842 100644 --- a/misc/simulators/simulate_load.py +++ b/misc/simulators/simulate_load.py @@ -4,6 +4,8 @@ from __future__ import print_function +from past.builtins import cmp + import random SERVER_CAPACITY = 10**12 diff --git a/misc/simulators/sizes.py b/misc/simulators/sizes.py index 1719700fa..eb5f3adbf 100644 --- a/misc/simulators/sizes.py +++ b/misc/simulators/sizes.py @@ -2,6 +2,11 @@ from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import input + + import random, math, re from twisted.python import usage @@ -205,7 +210,7 @@ def graph(): series["alacrity"][file_size] = s.bytes_until_some_data g.plot([ (fs, series["overhead"][fs]) for fs in sizes ]) - raw_input("press return") + input("press return") if __name__ == '__main__': diff --git a/newsfragments/1792.feature b/newsfragments/1792.feature deleted file mode 100644 index b2b839664..000000000 --- a/newsfragments/1792.feature +++ /dev/null @@ -1 +0,0 @@ -PyPy is now a supported platform. \ No newline at end of file diff --git a/newsfragments/2755.other b/newsfragments/2755.other deleted file mode 100644 index e3e31465d..000000000 --- a/newsfragments/2755.other +++ /dev/null @@ -1 +0,0 @@ -The Tahoe-LAFS project has adopted a formal code of conduct. diff --git a/newsfragments/3037.other b/newsfragments/3037.other new file mode 100644 index 000000000..947dc8f60 --- /dev/null +++ b/newsfragments/3037.other @@ -0,0 +1 @@ +The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs. \ No newline at end of file diff --git a/newsfragments/3284.removed b/newsfragments/3284.removed deleted file mode 100644 index 7e31d352a..000000000 --- a/newsfragments/3284.removed +++ /dev/null @@ -1 +0,0 @@ -The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at . diff --git a/newsfragments/3296.installation b/newsfragments/3296.installation deleted file mode 100644 index 78cf83f60..000000000 --- a/newsfragments/3296.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7. \ No newline at end of file diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix deleted file mode 100644 index 9939fe1f0..000000000 --- a/newsfragments/3312.bugfix +++ /dev/null @@ -1 +0,0 @@ -Make directory page links work. diff --git a/newsfragments/3313.minor b/newsfragments/3313.minor deleted file mode 100644 index c4eecd956..000000000 --- a/newsfragments/3313.minor +++ /dev/null @@ -1 +0,0 @@ -Replace nevow with twisted.web in web.operations.OphandleTable diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor deleted file mode 100644 index 0536c297a..000000000 --- a/newsfragments/3315.minor +++ /dev/null @@ -1 +0,0 @@ -Replace nevow with twisted.web in web.operations.ReloadMixin diff --git a/newsfragments/3316.minor b/newsfragments/3316.minor deleted file mode 100644 index 9457b486e..000000000 --- a/newsfragments/3316.minor +++ /dev/null @@ -1 +0,0 @@ -Port checker result pages' rendering from nevow to twisted web templates. diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature deleted file mode 100644 index 2a7048397..000000000 --- a/newsfragments/3317.feature +++ /dev/null @@ -1 +0,0 @@ -allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. \ No newline at end of file diff --git a/newsfragments/3323.removed b/newsfragments/3323.removed deleted file mode 100644 index 356b4b2af..000000000 --- a/newsfragments/3323.removed +++ /dev/null @@ -1 +0,0 @@ -Slackware 14.2 is no longer a Tahoe-LAFS supported platform. diff --git a/newsfragments/3328.installation b/newsfragments/3328.installation deleted file mode 100644 index 7b08ffdc4..000000000 --- a/newsfragments/3328.installation +++ /dev/null @@ -1 +0,0 @@ -Tahoe-LAFS now supports Ubuntu 20.04. \ No newline at end of file diff --git a/newsfragments/3348.bugfix b/newsfragments/3348.bugfix deleted file mode 100644 index e0f1f6c5e..000000000 --- a/newsfragments/3348.bugfix +++ /dev/null @@ -1 +0,0 @@ -Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests. diff --git a/newsfragments/3349.bugfix b/newsfragments/3349.bugfix deleted file mode 100644 index 08f2d7314..000000000 --- a/newsfragments/3349.bugfix +++ /dev/null @@ -1 +0,0 @@ -Mutable files now use RSA exponent 65537 diff --git a/newsfragments/3354.minor b/newsfragments/3354.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3354.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3355.other b/newsfragments/3355.other deleted file mode 100644 index 4e854e4dd..000000000 --- a/newsfragments/3355.other +++ /dev/null @@ -1 +0,0 @@ -The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments. diff --git a/newsfragments/3357.minor b/newsfragments/3357.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3357.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor deleted file mode 100644 index 8b1378917..000000000 --- a/newsfragments/3372.minor +++ /dev/null @@ -1 +0,0 @@ - diff --git a/newsfragments/3247.minor b/newsfragments/3390.minor similarity index 100% rename from newsfragments/3247.minor rename to newsfragments/3390.minor diff --git a/newsfragments/3392.minor b/newsfragments/3392.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3393.minor b/newsfragments/3393.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3394.minor b/newsfragments/3394.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3396.minor b/newsfragments/3396.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3397.minor b/newsfragments/3397.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3398.minor b/newsfragments/3398.minor deleted file mode 100644 index 477c141fd..000000000 --- a/newsfragments/3398.minor +++ /dev/null @@ -1 +0,0 @@ -Added pre-commit config to run flake8 checks on commit/push. diff --git a/newsfragments/3401.minor b/newsfragments/3401.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3403.minor b/newsfragments/3403.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3254.minor b/newsfragments/3404.minor similarity index 100% rename from newsfragments/3254.minor rename to newsfragments/3404.minor diff --git a/newsfragments/3406.minor b/newsfragments/3406.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3408.minor b/newsfragments/3408.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3409.minor b/newsfragments/3409.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3411.minor b/newsfragments/3411.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3415.minor b/newsfragments/3415.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3416.minor b/newsfragments/3416.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3417.minor b/newsfragments/3417.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3421.minor b/newsfragments/3421.minor deleted file mode 100644 index d6f70f6d9..000000000 --- a/newsfragments/3421.minor +++ /dev/null @@ -1 +0,0 @@ -Various, minor development `./Makefile` cleanup and improvement. diff --git a/newsfragments/3422.minor b/newsfragments/3422.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3423.minor b/newsfragments/3423.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3424.minor b/newsfragments/3424.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3425.minor b/newsfragments/3425.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3426.minor b/newsfragments/3426.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3427.minor b/newsfragments/3427.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3429.minor b/newsfragments/3429.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3430.minor b/newsfragments/3430.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3431.minor b/newsfragments/3431.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3436.minor b/newsfragments/3436.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3437.minor b/newsfragments/3437.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3438.minor b/newsfragments/3438.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3439.minor b/newsfragments/3439.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3440.minor b/newsfragments/3440.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3442.minor b/newsfragments/3442.minor deleted file mode 100644 index d67c7fb62..000000000 --- a/newsfragments/3442.minor +++ /dev/null @@ -1 +0,0 @@ -Minor test runner improvements and docs. diff --git a/newsfragments/3443.minor b/newsfragments/3443.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3446.minor b/newsfragments/3446.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3448.minor b/newsfragments/3448.minor deleted file mode 100644 index 4a5f7243f..000000000 --- a/newsfragments/3448.minor +++ /dev/null @@ -1 +0,0 @@ -Convert modules that only reference `unicode` to use `str`. \ No newline at end of file diff --git a/newsfragments/3449.minor b/newsfragments/3449.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3450.minor b/newsfragments/3450.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3451.minor b/newsfragments/3451.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3452.minor b/newsfragments/3452.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3453.minor b/newsfragments/3453.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3455.minor b/newsfragments/3455.minor deleted file mode 100644 index d7af32b64..000000000 --- a/newsfragments/3455.minor +++ /dev/null @@ -1 +0,0 @@ -Begin porting the `node` module to Python 3. diff --git a/newsfragments/3456.minor b/newsfragments/3456.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3458.minor b/newsfragments/3458.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3462.minor b/newsfragments/3462.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3463.minor b/newsfragments/3463.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/newsfragments/3464.minor b/newsfragments/3464.minor deleted file mode 100644 index bc79dee53..000000000 --- a/newsfragments/3464.minor +++ /dev/null @@ -1 +0,0 @@ -Cleanup comments that don't match the project convention. diff --git a/newsfragments/3263.other b/newsfragments/3563.minor similarity index 100% rename from newsfragments/3263.other rename to newsfragments/3563.minor diff --git a/newsfragments/3277.minor b/newsfragments/3616.minor similarity index 100% rename from newsfragments/3277.minor rename to newsfragments/3616.minor diff --git a/newsfragments/3278.minor b/newsfragments/3619.minor similarity index 100% rename from newsfragments/3278.minor rename to newsfragments/3619.minor diff --git a/newsfragments/3287.minor b/newsfragments/3626.minor similarity index 100% rename from newsfragments/3287.minor rename to newsfragments/3626.minor diff --git a/newsfragments/3288.minor b/newsfragments/3630.minor similarity index 100% rename from newsfragments/3288.minor rename to newsfragments/3630.minor diff --git a/newsfragments/3289.minor b/newsfragments/3632.minor similarity index 100% rename from newsfragments/3289.minor rename to newsfragments/3632.minor diff --git a/newsfragments/3290.minor b/newsfragments/3645.minor similarity index 100% rename from newsfragments/3290.minor rename to newsfragments/3645.minor diff --git a/newsfragments/3291.minor b/newsfragments/3647.minor similarity index 100% rename from newsfragments/3291.minor rename to newsfragments/3647.minor diff --git a/newsfragments/3292.minor b/newsfragments/3648.minor similarity index 100% rename from newsfragments/3292.minor rename to newsfragments/3648.minor diff --git a/newsfragments/3293.minor b/newsfragments/3649.minor similarity index 100% rename from newsfragments/3293.minor rename to newsfragments/3649.minor diff --git a/newsfragments/3650.bugfix b/newsfragments/3650.bugfix new file mode 100644 index 000000000..09a810239 --- /dev/null +++ b/newsfragments/3650.bugfix @@ -0,0 +1 @@ +``tahoe invite`` will now read share encoding/placement configuration values from a Tahoe client node configuration file if they are not given on the command line, instead of raising an unhandled exception. diff --git a/newsfragments/3651.minor b/newsfragments/3651.minor new file mode 100644 index 000000000..9a2f5a0ed --- /dev/null +++ b/newsfragments/3651.minor @@ -0,0 +1 @@ +We added documentation detailing the project's ticket triage process diff --git a/newsfragments/3652.removed b/newsfragments/3652.removed new file mode 100644 index 000000000..a3e964702 --- /dev/null +++ b/newsfragments/3652.removed @@ -0,0 +1 @@ +Removed support for the Account Server frontend authentication type. diff --git a/newsfragments/3294.minor b/newsfragments/3653.minor similarity index 100% rename from newsfragments/3294.minor rename to newsfragments/3653.minor diff --git a/newsfragments/3297.minor b/newsfragments/3654.minor similarity index 100% rename from newsfragments/3297.minor rename to newsfragments/3654.minor diff --git a/newsfragments/3298.minor b/newsfragments/3655.minor similarity index 100% rename from newsfragments/3298.minor rename to newsfragments/3655.minor diff --git a/newsfragments/3299.minor b/newsfragments/3656.minor similarity index 100% rename from newsfragments/3299.minor rename to newsfragments/3656.minor diff --git a/newsfragments/3300.minor b/newsfragments/3657.minor similarity index 100% rename from newsfragments/3300.minor rename to newsfragments/3657.minor diff --git a/newsfragments/3302.minor b/newsfragments/3658.minor similarity index 100% rename from newsfragments/3302.minor rename to newsfragments/3658.minor diff --git a/newsfragments/3303.minor b/newsfragments/3659.documentation similarity index 100% rename from newsfragments/3303.minor rename to newsfragments/3659.documentation diff --git a/newsfragments/3304.minor b/newsfragments/3662.minor similarity index 100% rename from newsfragments/3304.minor rename to newsfragments/3662.minor diff --git a/newsfragments/3663.other b/newsfragments/3663.other new file mode 100644 index 000000000..62abf2666 --- /dev/null +++ b/newsfragments/3663.other @@ -0,0 +1 @@ +You can run `make livehtml` in docs directory to invoke sphinx-autobuild. diff --git a/newsfragments/3664.documentation b/newsfragments/3664.documentation new file mode 100644 index 000000000..ab5de8884 --- /dev/null +++ b/newsfragments/3664.documentation @@ -0,0 +1 @@ +Documentation now has its own towncrier category. diff --git a/newsfragments/3666.documentation b/newsfragments/3666.documentation new file mode 100644 index 000000000..3f9e34777 --- /dev/null +++ b/newsfragments/3666.documentation @@ -0,0 +1 @@ +`tox -e docs` will treat warnings about docs as errors. diff --git a/newsfragments/3305.minor b/newsfragments/3667.minor similarity index 100% rename from newsfragments/3305.minor rename to newsfragments/3667.minor diff --git a/newsfragments/3306.minor b/newsfragments/3669.minor similarity index 100% rename from newsfragments/3306.minor rename to newsfragments/3669.minor diff --git a/newsfragments/3308.minor b/newsfragments/3670.minor similarity index 100% rename from newsfragments/3308.minor rename to newsfragments/3670.minor diff --git a/newsfragments/3309.minor b/newsfragments/3671.minor similarity index 100% rename from newsfragments/3309.minor rename to newsfragments/3671.minor diff --git a/newsfragments/3320.minor b/newsfragments/3672.minor similarity index 100% rename from newsfragments/3320.minor rename to newsfragments/3672.minor diff --git a/newsfragments/3324.other b/newsfragments/3674.minor similarity index 100% rename from newsfragments/3324.other rename to newsfragments/3674.minor diff --git a/newsfragments/3325.minor b/newsfragments/3675.minor similarity index 100% rename from newsfragments/3325.minor rename to newsfragments/3675.minor diff --git a/newsfragments/3326.minor b/newsfragments/3676.minor similarity index 100% rename from newsfragments/3326.minor rename to newsfragments/3676.minor diff --git a/newsfragments/3677.documentation b/newsfragments/3677.documentation new file mode 100644 index 000000000..51730e765 --- /dev/null +++ b/newsfragments/3677.documentation @@ -0,0 +1 @@ +The visibility of the Tahoe-LAFS logo has been improved for "dark" themed viewing. diff --git a/newsfragments/3329.minor b/newsfragments/3678.minor similarity index 100% rename from newsfragments/3329.minor rename to newsfragments/3678.minor diff --git a/newsfragments/3330.minor b/newsfragments/3679.minor similarity index 100% rename from newsfragments/3330.minor rename to newsfragments/3679.minor diff --git a/newsfragments/3681.minor b/newsfragments/3681.minor new file mode 100644 index 000000000..bc84b6b8f --- /dev/null +++ b/newsfragments/3681.minor @@ -0,0 +1,8 @@ +(The below text is no longer valid: netifaces has released a 64-bit +Python 2.7 wheel for Windows. Ticket #3733 made the switch in CI. We +should be able to test and run Tahoe-LAFS without needing vcpython27 +now.) + +Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has +removed vcpython27 compiler downloads from their site, and Tahoe-LAFS +needs vcpython27 to build and install netifaces on 64-bit Windows. diff --git a/newsfragments/3682.documentation b/newsfragments/3682.documentation new file mode 100644 index 000000000..5cf78bd90 --- /dev/null +++ b/newsfragments/3682.documentation @@ -0,0 +1 @@ +A cheatsheet-style document for contributors was created at CONTRIBUTORS.rst \ No newline at end of file diff --git a/newsfragments/3331.minor b/newsfragments/3683.minor similarity index 100% rename from newsfragments/3331.minor rename to newsfragments/3683.minor diff --git a/newsfragments/3332.minor b/newsfragments/3686.minor similarity index 100% rename from newsfragments/3332.minor rename to newsfragments/3686.minor diff --git a/newsfragments/3333.minor b/newsfragments/3687.minor similarity index 100% rename from newsfragments/3333.minor rename to newsfragments/3687.minor diff --git a/newsfragments/3334.minor b/newsfragments/3691.minor similarity index 100% rename from newsfragments/3334.minor rename to newsfragments/3691.minor diff --git a/newsfragments/3335.minor b/newsfragments/3692.minor similarity index 100% rename from newsfragments/3335.minor rename to newsfragments/3692.minor diff --git a/newsfragments/3336.minor b/newsfragments/3699.minor similarity index 100% rename from newsfragments/3336.minor rename to newsfragments/3699.minor diff --git a/newsfragments/3338.minor b/newsfragments/3700.minor similarity index 100% rename from newsfragments/3338.minor rename to newsfragments/3700.minor diff --git a/newsfragments/3339.minor b/newsfragments/3701.minor similarity index 100% rename from newsfragments/3339.minor rename to newsfragments/3701.minor diff --git a/newsfragments/3340.minor b/newsfragments/3702.minor similarity index 100% rename from newsfragments/3340.minor rename to newsfragments/3702.minor diff --git a/newsfragments/3341.minor b/newsfragments/3703.minor similarity index 100% rename from newsfragments/3341.minor rename to newsfragments/3703.minor diff --git a/newsfragments/3342.minor b/newsfragments/3704.minor similarity index 100% rename from newsfragments/3342.minor rename to newsfragments/3704.minor diff --git a/newsfragments/3343.minor b/newsfragments/3705.minor similarity index 100% rename from newsfragments/3343.minor rename to newsfragments/3705.minor diff --git a/newsfragments/3344.minor b/newsfragments/3707.minor similarity index 100% rename from newsfragments/3344.minor rename to newsfragments/3707.minor diff --git a/newsfragments/3346.minor b/newsfragments/3708.minor similarity index 100% rename from newsfragments/3346.minor rename to newsfragments/3708.minor diff --git a/newsfragments/3351.minor b/newsfragments/3709.minor similarity index 100% rename from newsfragments/3351.minor rename to newsfragments/3709.minor diff --git a/newsfragments/3353.minor b/newsfragments/3711.minor similarity index 100% rename from newsfragments/3353.minor rename to newsfragments/3711.minor diff --git a/newsfragments/3712.installation b/newsfragments/3712.installation new file mode 100644 index 000000000..b80e1558b --- /dev/null +++ b/newsfragments/3712.installation @@ -0,0 +1 @@ +The Nix package now includes correct version information. \ No newline at end of file diff --git a/newsfragments/3356.minor b/newsfragments/3713.minor similarity index 100% rename from newsfragments/3356.minor rename to newsfragments/3713.minor diff --git a/newsfragments/3358.minor b/newsfragments/3714.minor similarity index 100% rename from newsfragments/3358.minor rename to newsfragments/3714.minor diff --git a/newsfragments/3359.minor b/newsfragments/3715.minor similarity index 100% rename from newsfragments/3359.minor rename to newsfragments/3715.minor diff --git a/newsfragments/3716.incompat b/newsfragments/3716.incompat new file mode 100644 index 000000000..aa03eea47 --- /dev/null +++ b/newsfragments/3716.incompat @@ -0,0 +1 @@ +tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. \ No newline at end of file diff --git a/newsfragments/3361.minor b/newsfragments/3717.minor similarity index 100% rename from newsfragments/3361.minor rename to newsfragments/3717.minor diff --git a/newsfragments/3364.minor b/newsfragments/3718.minor similarity index 100% rename from newsfragments/3364.minor rename to newsfragments/3718.minor diff --git a/newsfragments/3721.documentation b/newsfragments/3721.documentation new file mode 100644 index 000000000..36ae33236 --- /dev/null +++ b/newsfragments/3721.documentation @@ -0,0 +1 @@ +Our IRC channel, #tahoe-lafs, has been moved to irc.libera.chat. diff --git a/newsfragments/3365.minor b/newsfragments/3722.minor similarity index 100% rename from newsfragments/3365.minor rename to newsfragments/3722.minor diff --git a/newsfragments/3366.minor b/newsfragments/3723.minor similarity index 100% rename from newsfragments/3366.minor rename to newsfragments/3723.minor diff --git a/newsfragments/3726.documentation b/newsfragments/3726.documentation new file mode 100644 index 000000000..fb94fff32 --- /dev/null +++ b/newsfragments/3726.documentation @@ -0,0 +1 @@ +Tahoe-LAFS project is now registered with Libera.Chat IRC network. diff --git a/newsfragments/3367.minor b/newsfragments/3727.minor similarity index 100% rename from newsfragments/3367.minor rename to newsfragments/3727.minor diff --git a/newsfragments/3368.minor b/newsfragments/3728.minor similarity index 100% rename from newsfragments/3368.minor rename to newsfragments/3728.minor diff --git a/newsfragments/3370.minor b/newsfragments/3729.minor similarity index 100% rename from newsfragments/3370.minor rename to newsfragments/3729.minor diff --git a/newsfragments/3373.minor b/newsfragments/3730.minor similarity index 100% rename from newsfragments/3373.minor rename to newsfragments/3730.minor diff --git a/newsfragments/3374.minor b/newsfragments/3731.minor similarity index 100% rename from newsfragments/3374.minor rename to newsfragments/3731.minor diff --git a/newsfragments/3375.minor b/newsfragments/3732.minor similarity index 100% rename from newsfragments/3375.minor rename to newsfragments/3732.minor diff --git a/newsfragments/3733.installation b/newsfragments/3733.installation new file mode 100644 index 000000000..c1cac649b --- /dev/null +++ b/newsfragments/3733.installation @@ -0,0 +1 @@ +Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows. VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult. diff --git a/newsfragments/3376.minor b/newsfragments/3734.minor similarity index 100% rename from newsfragments/3376.minor rename to newsfragments/3734.minor diff --git a/newsfragments/3377.minor b/newsfragments/3735.minor similarity index 100% rename from newsfragments/3377.minor rename to newsfragments/3735.minor diff --git a/newsfragments/3378.minor b/newsfragments/3736.minor similarity index 100% rename from newsfragments/3378.minor rename to newsfragments/3736.minor diff --git a/newsfragments/3738.bugfix b/newsfragments/3738.bugfix new file mode 100644 index 000000000..6a4bc1cd9 --- /dev/null +++ b/newsfragments/3738.bugfix @@ -0,0 +1 @@ +Fix regression where uploading files with non-ASCII names failed. \ No newline at end of file diff --git a/newsfragments/3739.bugfix b/newsfragments/3739.bugfix new file mode 100644 index 000000000..875941cf8 --- /dev/null +++ b/newsfragments/3739.bugfix @@ -0,0 +1 @@ +Fixed annoying UnicodeWarning message on Python 2 when running CLI tools. \ No newline at end of file diff --git a/newsfragments/3380.minor b/newsfragments/3741.minor similarity index 100% rename from newsfragments/3380.minor rename to newsfragments/3741.minor diff --git a/newsfragments/3381.minor b/newsfragments/3743.minor similarity index 100% rename from newsfragments/3381.minor rename to newsfragments/3743.minor diff --git a/newsfragments/3382.minor b/newsfragments/3744.minor similarity index 100% rename from newsfragments/3382.minor rename to newsfragments/3744.minor diff --git a/newsfragments/3383.minor b/newsfragments/3745.minor similarity index 100% rename from newsfragments/3383.minor rename to newsfragments/3745.minor diff --git a/newsfragments/3386.minor b/newsfragments/3746.minor similarity index 100% rename from newsfragments/3386.minor rename to newsfragments/3746.minor diff --git a/newsfragments/3747.documentation b/newsfragments/3747.documentation new file mode 100644 index 000000000..a2559a6a0 --- /dev/null +++ b/newsfragments/3747.documentation @@ -0,0 +1 @@ +Rewriting the installation guide for Tahoe-LAFS. diff --git a/newsfragments/3387.minor b/newsfragments/3751.minor similarity index 100% rename from newsfragments/3387.minor rename to newsfragments/3751.minor diff --git a/newsfragments/3757.other b/newsfragments/3757.other new file mode 100644 index 000000000..3d2d3f272 --- /dev/null +++ b/newsfragments/3757.other @@ -0,0 +1 @@ +Refactored test_introducer in web tests to use custom base test cases \ No newline at end of file diff --git a/newsfragments/3388.minor b/newsfragments/3759.minor similarity index 100% rename from newsfragments/3388.minor rename to newsfragments/3759.minor diff --git a/newsfragments/3389.minor b/newsfragments/3760.minor similarity index 100% rename from newsfragments/3389.minor rename to newsfragments/3760.minor diff --git a/newsfragments/3391.minor b/newsfragments/3763.minor similarity index 100% rename from newsfragments/3391.minor rename to newsfragments/3763.minor diff --git a/newsfragments/3764.documentation b/newsfragments/3764.documentation new file mode 100644 index 000000000..d473cd27c --- /dev/null +++ b/newsfragments/3764.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. \ No newline at end of file diff --git a/newsfragments/3765.documentation b/newsfragments/3765.documentation new file mode 100644 index 000000000..a3b59c4d6 --- /dev/null +++ b/newsfragments/3765.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes a glossary. \ No newline at end of file diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 6c3c68343..35b29f1cc 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -7,7 +7,20 @@ , html5lib, pyutil, distro, configparser }: python.pkgs.buildPythonPackage rec { - version = "1.14.0.dev"; + # Most of the time this is not exactly the release version (eg 1.15.1). + # Give it a `post` component to make it look newer than the release version + # and we'll bump this up at the time of each release. + # + # It's difficult to read the version from Git the way the Python code does + # for two reasons. First, doing so involves populating the Nix expression + # with values from the source. Nix calls this "import from derivation" or + # "IFD" (). This is + # discouraged in most cases - including this one, I think. Second, the + # Python code reads the contents of `.git` to determine its version. `.git` + # is not a reproducable artifact (in the sense of "reproducable builds") so + # it is excluded from the source tree by default. When it is included, the + # package tends to be frequently spuriously rebuilt. + version = "1.15.1.post1"; name = "tahoe-lafs-${version}"; src = lib.cleanSourceWith { src = ../.; @@ -22,20 +35,22 @@ python.pkgs.buildPythonPackage rec { # Build up a bunch of knowledge about what kind of file this is. isTox = type == "directory" && basename == ".tox"; isTrialTemp = type == "directory" && basename == "_trial_temp"; - isVersion = basename == "version.py"; + isVersion = basename == "_version.py"; isBytecode = ext == "pyc" || ext == "pyo"; isBackup = lib.hasSuffix "~" basename; isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename; isSymlink = type == "symlink"; + isGit = type == "directory" && basename == ".git"; in # Exclude all these things - ! (isTrialTemp - || isTox + ! (isTox + || isTrialTemp || isVersion || isBytecode || isBackup || isTemporary || isSymlink + || isGit ); }; @@ -57,10 +72,20 @@ python.pkgs.buildPythonPackage rec { rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py - # Since we're deleting files, this complains they're missing. For now Nix - # is Python 2-only, anyway, so these tests don't add anything yet. - rm src/allmydata/test/test_python3.py - ''; + # Generate _version.py ourselves since we can't rely on the Python code + # extracting the information from the .git directory we excluded. + cat > src/allmydata/_version.py <= 0.12.5 has ConnectionInfo and ReconnectionInfo # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely - # * foolscap >= 20.4 is necessary for Python 3 + # * foolscap >= 21.7 is necessary for Python 3 with i2p support. "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 20.4.0 ; python_version > '3.0'", + "foolscap >= 21.7.0 ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -113,12 +114,11 @@ install_requires = [ # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped # Python 2 entirely; stick to the version known to work for us. - # XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404 - "pyrsistent < 0.17.0", + "pyrsistent < 0.17.0 ; python_version < '3.0'", + "pyrsistent ; python_version > '3.0'", # A great way to define types of values. - # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390 - "attrs >= 18.2.0, < 20", + "attrs >= 18.2.0", # WebSocket library for twisted and asyncio "autobahn >= 19.5.2", @@ -357,7 +357,7 @@ if version: setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'rU').read(), + long_description=open('README.rst', 'r', encoding='utf-8').read(), author='the Tahoe-LAFS project', author_email='tahoe-dev@tahoe-lafs.org', url='https://tahoe-lafs.org/', @@ -389,6 +389,10 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", + # XXX: decorator isn't a direct dependency, but pytest-twisted + # depends on decorator, and decorator 5.x isn't compatible with + # Python 2.7. + "decorator < 5", "hypothesis >= 3.6.1", "treq", "towncrier", diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 3157c8c80..333394fc5 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -3,6 +3,16 @@ Decentralized storage grid. community web site: U{https://tahoe-lafs.org/} """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + # Don't import future str() so we don't break Foolscap serialization on Python 2. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + from past.builtins import unicode as str __all__ = [ "__version__", @@ -52,3 +62,18 @@ standard_library.install_aliases() from ._monkeypatch import patch patch() del patch + + +# On Python 3, turn BytesWarnings into exceptions. This can have potential +# production impact... if BytesWarnings are actually present in the codebase. +# Given that this has been enabled before Python 3 Tahoe-LAFS was publicly +# released, no such code should exist, and this will ensure it doesn't get +# added either. +# +# Also note that BytesWarnings only happen if Python is run with -b option, so +# in practice this should only affect tests. +if PY3: + import warnings + # Error on BytesWarnings, to catch things like str(b""), but only for + # allmydata code. + warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*") diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 570d4723e..43eb36cc6 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -142,7 +142,7 @@ class ProhibitedNode(object): def get_best_readable_version(self): raise FileProhibited(self.reason) - def download_best_version(self, progress=None): + def download_best_version(self): raise FileProhibited(self.reason) def get_best_mutable_version(self): diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 17475a951..ac2006924 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -123,7 +123,6 @@ _client_config = configutil.ValidConfiguration( ), "sftpd": ( "accounts.file", - "accounts.url", "enabled", "host_privkey_file", "host_pubkey_file", @@ -949,7 +948,8 @@ class _Client(node.Node, pollmixin.PollMixin): random data in "api_auth_token" which must be echoed to API calls. """ - return self.config.get_private_config('api_auth_token') + return self.config.get_private_config( + 'api_auth_token').encode("ascii") def _create_auth_token(self): """ @@ -1062,13 +1062,12 @@ class _Client(node.Node, pollmixin.PollMixin): accountfile = self.config.get_config("sftpd", "accounts.file", None) if accountfile: accountfile = self.config.get_config_path(accountfile) - accounturl = self.config.get_config("sftpd", "accounts.url", None) sftp_portstr = self.config.get_config("sftpd", "port", "tcp:8022") pubkey_file = self.config.get_config("sftpd", "host_pubkey_file") privkey_file = self.config.get_config("sftpd", "host_privkey_file") from allmydata.frontends import sftpd - s = sftpd.SFTPServer(self, accountfile, accounturl, + s = sftpd.SFTPServer(self, accountfile, sftp_portstr, pubkey_file, privkey_file) s.setServiceParent(self) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index ea8e84e05..fdf373b45 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -646,7 +646,7 @@ class DirectoryNode(object): return d - def add_file(self, namex, uploadable, metadata=None, overwrite=True, progress=None): + def add_file(self, namex, uploadable, metadata=None, overwrite=True): """I upload a file (using the given IUploadable), then attach the resulting FileNode to the directory at the given name. I return a Deferred that fires (with the IFileNode of the uploaded file) when @@ -657,7 +657,7 @@ class DirectoryNode(object): d = DeferredContext(defer.fail(NotWriteableError())) else: # XXX should pass reactor arg - d = DeferredContext(self._uploader.upload(uploadable, progress=progress)) + d = DeferredContext(self._uploader.upload(uploadable)) d.addCallback(lambda results: self._create_and_validate_node(results.get_uri(), None, name)) diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py index 7f81572fe..b61062334 100644 --- a/src/allmydata/frontends/auth.py +++ b/src/allmydata/frontends/auth.py @@ -1,14 +1,22 @@ -import os +""" +Authentication for frontends. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer -from twisted.web.client import getPage from twisted.internet import defer from twisted.cred import error, checkers, credentials from twisted.conch.ssh import keys from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB from allmydata.util.dictutil import BytesKeyDict -from allmydata.util import base32 from allmydata.util.fileutil import abspath_expanduser_unicode @@ -84,56 +92,5 @@ class AccountFileChecker(object): return defer.fail(error.UnauthorizedLogin()) d = defer.maybeDeferred(creds.checkPassword, correct) - d.addCallback(self._cbPasswordMatch, str(creds.username)) + d.addCallback(self._cbPasswordMatch, creds.username) return d - - -@implementer(checkers.ICredentialsChecker) -class AccountURLChecker(object): - credentialInterfaces = (credentials.IUsernamePassword,) - - def __init__(self, client, auth_url): - self.client = client - self.auth_url = auth_url - - def _cbPasswordMatch(self, rootcap, username): - return FTPAvatarID(username, rootcap) - - def post_form(self, username, password): - sepbase = base32.b2a(os.urandom(4)) - sep = "--" + sepbase - form = [] - form.append(sep) - fields = {"action": "authenticate", - "email": username, - "passwd": password, - } - for name, value in fields.iteritems(): - form.append('Content-Disposition: form-data; name="%s"' % name) - form.append('') - assert isinstance(value, str) - form.append(value) - form.append(sep) - form[-1] += "--" - body = "\r\n".join(form) + "\r\n" - headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase, - } - return getPage(self.auth_url, method="POST", - postdata=body, headers=headers, - followRedirect=True, timeout=30) - - def _parse_response(self, res): - rootcap = res.strip() - if rootcap == "0": - raise error.UnauthorizedLogin - return rootcap - - def requestAvatarId(self, credentials): - # construct a POST to the login form. While this could theoretically - # be done with something like the stdlib 'email' package, I can't - # figure out how, so we just slam together a form manually. - d = self.post_form(credentials.username, credentials.password) - d.addCallback(self._parse_response) - d.addCallback(self._cbPasswordMatch, str(credentials.username)) - return d - diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index bc7196de6..d2d614c77 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1011,8 +1011,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): PrefixingLogMixin.__init__(self, facility="tahoe.sftp", prefix=username) if noisy: self.log(".__init__(%r, %r, %r)" % (client, rootnode, username), level=NOISY) - self.channelLookup["session"] = session.SSHSession - self.subsystemLookup["sftp"] = FileTransferServer + self.channelLookup[b"session"] = session.SSHSession + self.subsystemLookup[b"sftp"] = FileTransferServer self._client = client self._root = rootnode @@ -1983,7 +1983,7 @@ class ShellSession(PrefixingLogMixin): components.registerAdapter(ShellSession, SFTPUserHandler, ISession) -from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme +from allmydata.frontends.auth import AccountFileChecker, NeedRootcapLookupScheme @implementer(portal.IRealm) class Dispatcher(object): @@ -2000,7 +2000,7 @@ class Dispatcher(object): class SFTPServer(service.MultiService): name = "frontend:sftp" - def __init__(self, client, accountfile, accounturl, + def __init__(self, client, accountfile, sftp_portstr, pubkey_file, privkey_file): precondition(isinstance(accountfile, (str, type(None))), accountfile) precondition(isinstance(pubkey_file, str), pubkey_file) @@ -2013,12 +2013,9 @@ class SFTPServer(service.MultiService): if accountfile: c = AccountFileChecker(self, accountfile) p.registerChecker(c) - if accounturl: - c = AccountURLChecker(self, accounturl) - p.registerChecker(c) - if not accountfile and not accounturl: + if not accountfile: # we could leave this anonymous, with just the /uri/CAP form - raise NeedRootcapLookupScheme("must provide an account file or URL") + raise NeedRootcapLookupScheme("must provide an account file") pubkey = keys.Key.fromFile(pubkey_file.encode(get_filesystem_encoding())) privkey = keys.Key.fromFile(privkey_file.encode(get_filesystem_encoding())) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index bbb7f8123..42fc18077 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -90,7 +90,7 @@ PiB=1024*TiB @implementer(IEncoder) class Encoder(object): - def __init__(self, log_parent=None, upload_status=None, progress=None): + def __init__(self, log_parent=None, upload_status=None): object.__init__(self) self.uri_extension_data = {} self._codec = None @@ -102,7 +102,6 @@ class Encoder(object): self._log_number = log.msg("creating Encoder %s" % self, facility="tahoe.encoder", parent=log_parent) self._aborted = False - self._progress = progress def __repr__(self): if hasattr(self, "_storage_index"): @@ -123,8 +122,6 @@ class Encoder(object): def _got_size(size): self.log(format="file size: %(size)d", size=size) self.file_size = size - if self._progress: - self._progress.set_progress_total(self.file_size) d.addCallback(_got_size) d.addCallback(lambda res: eu.get_all_encoding_parameters()) d.addCallback(self._got_all_encoding_parameters) @@ -462,13 +459,6 @@ class Encoder(object): dl = self._gather_responses(dl) - def do_progress(ign): - done = self.segment_size * (segnum + 1) - if self._progress: - self._progress.set_progress(done) - return ign - dl.addCallback(do_progress) - def _logit(res): self.log("%s uploaded %s / %s bytes (%d%%) of your file." % (self, diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 9e13e1337..6962c31a4 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -337,13 +337,13 @@ class ImmutableFileNode(object): """ return defer.succeed(self) - def download_best_version(self, progress=None): + def download_best_version(self): """ Download the best version of this file, returning its contents as a bytestring. Since there is only one version of an immutable file, we download and return the contents of this file. """ - d = consumer.download_to_data(self, progress=progress) + d = consumer.download_to_data(self) return d # for an immutable file, download_to_data (specified in IReadable) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 6ed5571b9..544a205e1 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -113,10 +113,7 @@ class LiteralFileNode(_ImmutableFileNodeBase): return defer.succeed(self) - def download_best_version(self, progress=None): - if progress is not None: - progress.set_progress_total(len(self.u.data)) - progress.set_progress(len(self.u.data)) + def download_best_version(self): return defer.succeed(self.u.data) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 2c4b9db78..8ce51782c 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -154,8 +154,8 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warn def __init__(self, storage_index, helper, storage_broker, secret_holder, incoming_file, encoding_file, - log_number, progress=None): - upload.CHKUploader.__init__(self, storage_broker, secret_holder, progress=progress) + log_number): + upload.CHKUploader.__init__(self, storage_broker, secret_holder) self._storage_index = storage_index self._helper = helper self._incoming_file = incoming_file diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e8896c10e..0b30354f6 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -48,7 +48,7 @@ from allmydata.util.rrefutil import add_version_to_remote_reference from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \ IEncryptedUploadable, RIEncryptedUploadable, IUploadStatus, \ NoServersError, InsufficientVersionError, UploadUnhappinessError, \ - DEFAULT_MAX_SEGMENT_SIZE, IProgress, IPeerSelector + DEFAULT_MAX_SEGMENT_SIZE, IPeerSelector from allmydata.immutable import layout from io import BytesIO @@ -945,7 +945,7 @@ class EncryptAnUploadable(object): IEncryptedUploadable.""" CHUNKSIZE = 50*1024 - def __init__(self, original, log_parent=None, progress=None, chunk_size=None): + def __init__(self, original, log_parent=None, chunk_size=None): """ :param chunk_size: The number of bytes to read from the uploadable at a time, or None for some default. @@ -962,7 +962,6 @@ class EncryptAnUploadable(object): self._file_size = None self._ciphertext_bytes_read = 0 self._status = None - self._progress = progress if chunk_size is not None: self.CHUNKSIZE = chunk_size @@ -985,8 +984,6 @@ class EncryptAnUploadable(object): self._file_size = size if self._status: self._status.set_size(size) - if self._progress: - self._progress.set_progress_total(size) return size d.addCallback(_got_size) return d @@ -1229,7 +1226,7 @@ class UploadStatus(object): class CHKUploader(object): - def __init__(self, storage_broker, secret_holder, progress=None, reactor=None): + def __init__(self, storage_broker, secret_holder, reactor=None): # server_selector needs storage_broker and secret_holder self._storage_broker = storage_broker self._secret_holder = secret_holder @@ -1239,7 +1236,6 @@ class CHKUploader(object): self._upload_status = UploadStatus() self._upload_status.set_helper(False) self._upload_status.set_active(True) - self._progress = progress self._reactor = reactor # locate_all_shareholders() will create the following attribute: @@ -1294,7 +1290,6 @@ class CHKUploader(object): self._encoder = encode.Encoder( self._log_number, self._upload_status, - progress=self._progress, ) # this just returns itself yield self._encoder.set_encrypted_uploadable(eu) @@ -1428,13 +1423,12 @@ def read_this_many_bytes(uploadable, size, prepend_data=[]): class LiteralUploader(object): - def __init__(self, progress=None): + def __init__(self): self._status = s = UploadStatus() s.set_storage_index(None) s.set_helper(False) s.set_progress(0, 1.0) s.set_active(False) - self._progress = progress def start(self, uploadable): uploadable = IUploadable(uploadable) @@ -1442,8 +1436,6 @@ class LiteralUploader(object): def _got_size(size): self._size = size self._status.set_size(size) - if self._progress: - self._progress.set_progress_total(size) return read_this_many_bytes(uploadable, size) d.addCallback(_got_size) d.addCallback(lambda data: uri.LiteralFileURI(b"".join(data))) @@ -1467,8 +1459,6 @@ class LiteralUploader(object): self._status.set_progress(1, 1.0) self._status.set_progress(2, 1.0) self._status.set_results(ur) - if self._progress: - self._progress.set_progress(self._size) return ur def close(self): @@ -1867,13 +1857,12 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): name = "uploader" URI_LIT_SIZE_THRESHOLD = 55 - def __init__(self, helper_furl=None, stats_provider=None, history=None, progress=None): + def __init__(self, helper_furl=None, stats_provider=None, history=None): self._helper_furl = helper_furl self.stats_provider = stats_provider self._history = history self._helper = None self._all_uploads = weakref.WeakKeyDictionary() # for debugging - self._progress = progress log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.upload") service.MultiService.__init__(self) @@ -1907,13 +1896,12 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): return (self._helper_furl, bool(self._helper)) - def upload(self, uploadable, progress=None, reactor=None): + def upload(self, uploadable, reactor=None): """ Returns a Deferred that will fire with the UploadResults instance. """ assert self.parent assert self.running - assert progress is None or IProgress.providedBy(progress) uploadable = IUploadable(uploadable) d = uploadable.get_size() @@ -1922,15 +1910,13 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): precondition(isinstance(default_params, dict), default_params) precondition("max_segment_size" in default_params, default_params) uploadable.set_default_encoding_parameters(default_params) - if progress: - progress.set_progress_total(size) if self.stats_provider: self.stats_provider.count('uploader.files_uploaded', 1) self.stats_provider.count('uploader.bytes_uploaded', size) if size <= self.URI_LIT_SIZE_THRESHOLD: - uploader = LiteralUploader(progress=progress) + uploader = LiteralUploader() return uploader.start(uploadable) else: eu = EncryptAnUploadable(uploadable, self._parentmsgid) @@ -1943,7 +1929,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin): else: storage_broker = self.parent.get_storage_broker() secret_holder = self.parent._secret_holder - uploader = CHKUploader(storage_broker, secret_holder, progress=progress, reactor=reactor) + uploader = CHKUploader(storage_broker, secret_holder, reactor=reactor) d2.addCallback(lambda x: uploader.start(eu)) self._all_uploads[uploader] = None diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 5c6907f56..b5c791a9e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -737,38 +737,6 @@ class MustNotBeUnknownRWError(CapConstraintError): """Cannot add an unknown child cap specified in a rw_uri field.""" -class IProgress(Interface): - """ - Remembers progress measured in arbitrary units. Users of these - instances must call ``set_progress_total`` at least once before - progress can be valid, and must use the same units for both - ``set_progress_total`` and ``set_progress calls``. - - See also: - :class:`allmydata.util.progress.PercentProgress` - """ - - progress = Attribute( - "Current amount of progress (in percentage)" - ) - - def set_progress(value): - """ - Sets the current amount of progress. - - Arbitrary units, but must match units used for - set_progress_total. - """ - - def set_progress_total(value): - """ - Sets the total amount of expected progress - - Arbitrary units, but must be same units as used when calling - set_progress() on this instance).. - """ - - class IReadable(Interface): """I represent a readable object -- either an immutable file, or a specific version of a mutable file. @@ -798,11 +766,9 @@ class IReadable(Interface): def get_size(): """Return the length (in bytes) of this readable object.""" - def download_to_data(progress=None): + def download_to_data(): """Download all of the file contents. I return a Deferred that fires with the contents as a byte string. - - :param progress: None or IProgress implementer """ def read(consumer, offset=0, size=None): @@ -1110,13 +1076,11 @@ class IFileNode(IFilesystemNode): the Deferred will errback with an UnrecoverableFileError. """ - def download_best_version(progress=None): + def download_best_version(): """Download the contents of the version that would be returned by get_best_readable_version(). This is equivalent to calling download_to_data() on the IReadable given by that method. - progress is anything that implements IProgress - I return a Deferred that fires with a byte string when the file has been fully downloaded. To support streaming download, use the 'read' method of IReadable. If no version is recoverable, @@ -1262,7 +1226,7 @@ class IMutableFileNode(IFileNode): everything) to get increased visibility. """ - def upload(new_contents, servermap, progress=None): + def upload(new_contents, servermap): """Replace the contents of the file with new ones. This requires a servermap that was previously updated with MODE_WRITE. @@ -1283,8 +1247,6 @@ class IMutableFileNode(IFileNode): operation. If I do not signal UncoordinatedWriteError, then I was able to write the new version without incident. - ``progress`` is either None or an IProgress provider - I return a Deferred that fires (with a PublishStatus object) when the publish has completed. I will update the servermap in-place with the location of all new shares. @@ -1475,14 +1437,12 @@ class IDirectoryNode(IFilesystemNode): equivalent to calling set_node() multiple times, but is much more efficient.""" - def add_file(name, uploadable, metadata=None, overwrite=True, progress=None): + def add_file(name, uploadable, metadata=None, overwrite=True): """I upload a file (using the given IUploadable), then attach the resulting ImmutableFileNode to the directory at the given name. I set metadata the same way as set_uri and set_node. The child name must be a unicode string. - ``progress`` either provides IProgress or is None - I return a Deferred that fires (with the IFileNode of the uploaded file) when the operation completes.""" diff --git a/src/allmydata/introducer/__init__.py b/src/allmydata/introducer/__init__.py index c4a644e76..bfc960e05 100644 --- a/src/allmydata/introducer/__init__.py +++ b/src/allmydata/introducer/__init__.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from allmydata.introducer.server import create_introducer diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index f67aad203..f6f70d861 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -11,9 +11,11 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re + +from foolscap.furl import decode_furl from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 -from allmydata.util import base32, rrefutil, jsonbytes as json +from allmydata.util import base32, jsonbytes as json def get_tubid_string_from_ann(ann): @@ -123,10 +125,10 @@ class AnnouncementDescriptor(object): self.service_name = ann_d["service-name"] self.version = ann_d.get("my-version", "") self.nickname = ann_d.get("nickname", u"") - (service_name, key_s) = index + (_, key_s) = index self.serverid = key_s furl = ann_d.get("anonymous-storage-FURL") if furl: - self.connection_hints = rrefutil.connection_hints_for_furl(furl) + _, self.connection_hints, _ = decode_furl(furl) else: self.connection_hints = [] diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index dcc2fd2c0..1e28f511b 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -24,11 +24,12 @@ except ImportError: from zope.interface import implementer from twisted.application import service from twisted.internet import defer +from twisted.internet.address import IPv4Address from twisted.python.failure import Failure from foolscap.api import Referenceable import allmydata from allmydata import node -from allmydata.util import log, rrefutil, dictutil +from allmydata.util import log, dictutil from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider from allmydata.introducer.interfaces import \ @@ -148,6 +149,15 @@ class _IntroducerNode(node.Node): ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir) ws.setServiceParent(self) + +def stringify_remote_address(rref): + remote = rref.getPeer() + if isinstance(remote, IPv4Address): + return "%s:%d" % (remote.host, remote.port) + # loopback is a non-IPv4Address + return str(remote) + + @implementer(RIIntroducerPublisherAndSubscriberService_v2) class IntroducerService(service.MultiService, Referenceable): name = "introducer" @@ -216,7 +226,7 @@ class IntroducerService(service.MultiService, Referenceable): # tubid will be None. Also, subscribers do not tell us which # pubkey they use; only publishers do that. tubid = rref.getRemoteTubID() or "?" - remote_address = rrefutil.stringify_remote_address(rref) + remote_address = stringify_remote_address(rref) # these three assume subscriber_info["version"]==0, but # should tolerate other versions nickname = subscriber_info.get("nickname", u"?") diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index baedacb23..cd8cb0dc7 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -418,21 +418,21 @@ class MutableFileNode(object): return d.addCallback(_get_version, version) - def download_best_version(self, progress=None): + def download_best_version(self): """ I return a Deferred that fires with the contents of the best version of this mutable file. """ - return self._do_serialized(self._download_best_version, progress=progress) + return self._do_serialized(self._download_best_version) - def _download_best_version(self, progress=None): + def _download_best_version(self): """ I am the serialized sibling of download_best_version. """ d = self.get_best_readable_version() d.addCallback(self._record_size) - d.addCallback(lambda version: version.download_to_data(progress=progress)) + d.addCallback(lambda version: version.download_to_data()) # It is possible that the download will fail because there # aren't enough shares to be had. If so, we will try again after @@ -447,7 +447,7 @@ class MutableFileNode(object): d = self.get_best_mutable_version() d.addCallback(self._record_size) - d.addCallback(lambda version: version.download_to_data(progress=progress)) + d.addCallback(lambda version: version.download_to_data()) return d d.addErrback(_maybe_retry) @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap, progress=None): + def upload(self, new_contents, servermap): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of @@ -951,13 +951,13 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme + def download_to_data(self, fetch_privkey=False): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. """ - c = consumer.MemoryConsumer(progress=progress) + c = consumer.MemoryConsumer() d = self.read(c, fetch_privkey=fetch_privkey) d.addCallback(lambda mc: b"".join(mc.chunks)) return d diff --git a/src/allmydata/node.py b/src/allmydata/node.py index b8ff490c1..0feaffb35 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -848,7 +848,7 @@ def _tub_portlocation(config, get_local_addresses_sync, allocate_tcp_port): tubport = _convert_tub_port(cfg_tubport) for port in tubport.split(","): - if port in ("0", "tcp:0"): + if port in ("0", "tcp:0", "tcp:port=0", "tcp:0:interface=127.0.0.1"): raise PortAssignmentRequired() if cfg_location is None: diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 48e853f1e..86209637e 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_binary import json try: @@ -35,8 +45,10 @@ def print_keypair(options): from allmydata.crypto import ed25519 out = options.stdout private_key, public_key = ed25519.create_signing_keypair() - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), + file=out) + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), + file=out) class DerivePubkeyOptions(BaseOptions): def parseArgs(self, privkey): @@ -58,9 +70,10 @@ def derive_pubkey(options): out = options.stdout from allmydata.crypto import ed25519 privkey_vs = options.privkey + privkey_vs = ensure_binary(privkey_vs) private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), file=out) + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 1bffbbfc3..c7827e56e 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path, sys, time, random, stat from allmydata.util.netstring import netstring diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 6c5641b41..55975b8c5 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,5 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + + import os.path, re, fnmatch try: @@ -36,7 +47,7 @@ class FileStoreOptions(BaseOptions): # compute a node-url from the existing options, put in self['node-url'] if self['node-url']: - if (not isinstance(self['node-url'], basestring) + if (not isinstance(self['node-url'], (bytes, str)) or not NODEURL_RE.match(self['node-url'])): msg = ("--node-url is required to be a string and look like " "\"http://HOSTNAMEORADDR:PORT\", not: %r" % @@ -224,7 +235,7 @@ class CpOptions(FileStoreOptions): def parseArgs(self, *args): if len(args) < 2: raise usage.UsageError("cp requires at least two arguments") - self.sources = map(argv_to_unicode, args[:-1]) + self.sources = [argv_to_unicode(arg) for arg in args[:-1]] self.destination = argv_to_unicode(args[-1]) synopsis = "[options] FROM.. TO" @@ -346,14 +357,15 @@ class BackupOptions(FileStoreOptions): exclude = self['exclude'] exclude.add(g) - def opt_exclude_from(self, filepath): + def opt_exclude_from_utf_8(self, filepath): """Ignore file matching glob patterns listed in file, one per line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = file(abs_filepath) - except: - raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath)) + exclude_file = open(abs_filepath, "r", encoding="utf-8") + except Exception as e: + raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( + quote_local_unicode_path(abs_filepath), e)) try: for line in exclude_file: self.opt_exclude(line) @@ -435,7 +447,7 @@ class CheckOptions(FileStoreOptions): ("add-lease", None, "Add/renew lease on all shares."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ @@ -452,7 +464,7 @@ class DeepCheckOptions(FileStoreOptions): ("verbose", "v", "Be noisy about what is happening."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ @@ -503,7 +515,7 @@ def list_aliases(options): def list_(options): from allmydata.scripts import tahoe_ls - rc = tahoe_ls.list(options) + rc = tahoe_ls.ls(options) return rc def get(options): diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 29342ec6b..0a9ab8714 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,20 @@ # coding: utf-8 +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from six import ensure_str + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +else: + from typing import Union + import os, sys, textwrap import codecs @@ -18,15 +31,10 @@ from yaml import ( safe_dump, ) -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 - from twisted.python import usage from allmydata.util.assertutil import precondition -from allmydata.util.encodingutil import unicode_to_url, quote_output, \ +from allmydata.util.encodingutil import quote_output, \ quote_local_unicode_path, argv_to_abspath from allmydata.scripts.default_nodedir import _default_nodedir @@ -274,18 +282,27 @@ def get_alias(aliases, path_unicode, default): return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] def escape_path(path): - # type: (str) -> str + # type: (Union[str,bytes]) -> str u""" Return path quoted to US-ASCII, valid URL characters. >>> path = u'/føö/bar/☃' >>> escaped = escape_path(path) - >>> str(escaped) - '/f%C3%B8%C3%B6/bar/%E2%98%83' - >>> escaped.encode('ascii').decode('ascii') == escaped - True + >>> escaped + u'/f%C3%B8%C3%B6/bar/%E2%98%83' """ - segments = path.split("/") - result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments]) - result = ensure_str(result, "ascii") + if isinstance(path, str): + path = path.encode("utf-8") + segments = path.split(b"/") + result = str( + b"/".join([ + urllib.parse.quote(s).encode("ascii") for s in segments + ]), + "ascii" + ) + # Eventually (i.e. as part of Python 3 port) we want this to always return + # Unicode strings. However, to reduce diff sizes in the short term it'll + # return native string (i.e. bytes) on Python 2. + if PY2: + result = result.encode("ascii").__native__() return result diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index a53f6baa8..95099a2eb 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os from io import BytesIO from six.moves import urllib, http_client @@ -84,11 +94,17 @@ def do_http(method, url, body=b""): def format_http_success(resp): - return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False)) + # ensure_text() shouldn't be necessary when Python 2 is dropped. + return quote_output( + "%s %s" % (resp.status, six.ensure_text(resp.reason)), + quotemarks=False) def format_http_error(msg, resp): - return "%s: %s %s\n%s" % (msg, resp.status, quote_output(resp.reason, quotemarks=False), - quote_output(resp.read(), quotemarks=False)) + # ensure_text() shouldn't be necessary when Python 2 is dropped. + return quote_output( + "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason), + six.ensure_text(resp.read())), + quotemarks=False) def check_http_error(resp, stderr): if resp.status < 200 or resp.status >= 300: diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index b589ba81f..4959ed391 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -11,7 +11,6 @@ if PY2: import io import os -import json try: from allmydata.scripts.types_ import ( @@ -36,7 +35,7 @@ from allmydata.scripts.common import ( from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding -from allmydata.util import fileutil, i2p_provider, iputil, tor_provider +from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json from wormhole import wormhole @@ -393,7 +392,7 @@ def _get_config_via_wormhole(config): "client-v1": {}, } } - wh.send_message(json.dumps(intro)) + wh.send_message(json.dumps_bytes(intro)) server_intro = yield wh.get_message() server_intro = json.loads(server_intro) @@ -450,12 +449,13 @@ def create_node(config): v = remote_config.get(k, None) if v is not None: # we're faking usually argv-supplied options :/ + v_orig = v if isinstance(v, str): v = v.encode(get_io_encoding()) config[k] = v if k not in sensitive_keys: if k not in ['shares-happy', 'shares-total', 'shares-needed']: - print(" {}: {}".format(k, v), file=out) + print(" {}: {}".format(k, v_orig), file=out) else: print(" {}: [sensitive data; see tahoe.cfg]".format(k), file=out) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index b8aeee91e..2d6ba4602 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,12 +1,20 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2, bchr +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + try: from allmydata.scripts.types_ import SubCommands except ImportError: pass -from future.utils import bchr -from past.builtins import unicode # do not import any allmydata modules at this level. Do that from inside # individual functions instead. @@ -94,7 +102,7 @@ def dump_immutable_chk_share(f, out, options): def to_string(v): if isinstance(v, bytes): - return unicode(v, "utf-8") + return str(v, "utf-8") else: return str(v) @@ -173,9 +181,9 @@ def format_expiration_time(expiration_time): remains = expiration_time - now when = "%ds" % remains if remains > 24*3600: - when += " (%d days)" % (remains / (24*3600)) + when += " (%d days)" % (remains // (24*3600)) elif remains > 3600: - when += " (%d hours)" % (remains / 3600) + when += " (%d hours)" % (remains // 3600) return when @@ -205,7 +213,7 @@ def dump_mutable_share(options): print(file=out) print("Mutable slot found:", file=out) print(" share_type: %s" % share_type, file=out) - print(" write_enabler: %s" % unicode(base32.b2a(WE), "utf-8"), file=out) + print(" write_enabler: %s" % str(base32.b2a(WE), "utf-8"), file=out) print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out) print(" num_extra_leases: %d" % num_extra_leases, file=out) print(" container_size: %d" % container_size, file=out) @@ -217,8 +225,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.expiration_time) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % unicode(base32.b2a(lease.renew_secret), "utf-8"), file=out) - print(" cancel_secret: %s" % unicode(base32.b2a(lease.cancel_secret), "utf-8"), file=out) + print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) + print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) @@ -266,8 +274,8 @@ def dump_SDMF_share(m, length, options): print(" SDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) - print(" IV: %s" % unicode(base32.b2a(IV), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) + print(" IV: %s" % str(base32.b2a(IV), "utf-8"), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) print(" segsize: %d" % segsize, file=out) @@ -360,7 +368,7 @@ def dump_MDMF_share(m, length, options): print(" MDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) #print(" IV: %s" % base32.b2a(IV), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) @@ -452,7 +460,7 @@ def dump_cap(options): from allmydata import uri from allmydata.util import base32 from base64 import b32decode - import urlparse, urllib + from urllib.parse import unquote, urlparse out = options.stdout cap = options.cap @@ -461,18 +469,18 @@ def dump_cap(options): nodeid = b32decode(options['nodeid'].upper()) secret = None if options['client-secret']: - secret = base32.a2b(options['client-secret']) + secret = base32.a2b(options['client-secret'].encode("ascii")) elif options['client-dir']: secretfile = os.path.join(options['client-dir'], "private", "secret") try: - secret = base32.a2b(open(secretfile, "r").read().strip()) + secret = base32.a2b(open(secretfile, "rb").read().strip()) except EnvironmentError: pass if cap.startswith("http"): - scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap) + scheme, netloc, path, params, query, fragment = urlparse(cap) assert path.startswith("/uri/") - cap = urllib.unquote(path[len("/uri/"):]) + cap = unquote(path[len("/uri/"):]) u = uri.from_string(cap) @@ -485,19 +493,19 @@ def _dump_secrets(storage_index, secret, nodeid, out): if secret: crs = hashutil.my_renewal_secret_hash(secret) - print(" client renewal secret:", base32.b2a(crs), file=out) + print(" client renewal secret:", str(base32.b2a(crs), "ascii"), file=out) frs = hashutil.file_renewal_secret_hash(crs, storage_index) - print(" file renewal secret:", base32.b2a(frs), file=out) + print(" file renewal secret:", str(base32.b2a(frs), "ascii"), file=out) if nodeid: renew = hashutil.bucket_renewal_secret_hash(frs, nodeid) - print(" lease renewal secret:", base32.b2a(renew), file=out) + print(" lease renewal secret:", str(base32.b2a(renew), "ascii"), file=out) ccs = hashutil.my_cancel_secret_hash(secret) - print(" client cancel secret:", base32.b2a(ccs), file=out) + print(" client cancel secret:", str(base32.b2a(ccs), "ascii"), file=out) fcs = hashutil.file_cancel_secret_hash(ccs, storage_index) - print(" file cancel secret:", base32.b2a(fcs), file=out) + print(" file cancel secret:", str(base32.b2a(fcs), "ascii"), file=out) if nodeid: cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid) - print(" lease cancel secret:", base32.b2a(cancel), file=out) + print(" lease cancel secret:", str(base32.b2a(cancel), "ascii"), file=out) def dump_uri_instance(u, nodeid, secret, out, show_header=True): from allmydata import uri @@ -508,19 +516,19 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): if isinstance(u, uri.CHKFileURI): if show_header: print("CHK File:", file=out) - print(" key:", base32.b2a(u.key), file=out) - print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out) + print(" key:", str(base32.b2a(u.key), "ascii"), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.CHKFileVerifierURI): if show_header: print("CHK Verifier URI:", file=out) - print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) elif isinstance(u, uri.LiteralFileURI): if show_header: @@ -530,52 +538,52 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): elif isinstance(u, uri.WriteableSSKFileURI): # SDMF if show_header: print("SDMF Writeable URI:", file=out) - print(" writekey:", base32.b2a(u.writekey), file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", base32.b2a(we), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlySSKFileURI): if show_header: print("SDMF Read-only URI:", file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.SSKVerifierURI): if show_header: print("SDMF Verifier URI:", file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF if show_header: print("MDMF Writeable URI:", file=out) - print(" writekey:", base32.b2a(u.writekey), file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", base32.b2a(we), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlyMDMFFileURI): if show_header: print("MDMF Read-only URI:", file=out) - print(" readkey:", base32.b2a(u.readkey), file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.MDMFVerifierURI): if show_header: print("MDMF Verifier URI:", file=out) - print(" storage index:", si_b2a(u.get_storage_index()), file=out) - print(" fingerprint:", base32.b2a(u.fingerprint), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory @@ -623,7 +631,7 @@ class FindSharesOptions(BaseOptions): def parseArgs(self, storage_index_s, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath self.si_s = storage_index_s - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) description = """ Locate all shares for the given storage index. This command looks through one @@ -666,7 +674,7 @@ def find_shares(options): class CatalogSharesOptions(BaseOptions): def parseArgs(self, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) if not nodedirs: raise usage.UsageError("must specify at least one node directory") @@ -753,7 +761,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy @@ -782,7 +790,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) @@ -816,7 +824,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - unicode(ueb_hash, "utf-8"), expiration, + str(ueb_hash, "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: @@ -990,7 +998,7 @@ def fixOptionsClass(args): class FlogtoolOptions(foolscap_cli.Options): def __init__(self): super(FlogtoolOptions, self).__init__() - self.subCommands = map(fixOptionsClass, self.subCommands) + self.subCommands = list(map(fixOptionsClass, self.subCommands)) def getSynopsis(self): return "Usage: tahoe [global-options] debug flogtool COMMAND [flogtool-options]" diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index c38e20fa7..00924b8f9 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import six diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 4e6ff884d..fae2fcddd 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -1,6 +1,12 @@ from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -import warnings import os, sys from six.moves import StringIO import six @@ -16,7 +22,7 @@ from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ admin, tahoe_run, tahoe_invite -from allmydata.util.encodingutil import quote_local_unicode_path +from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode from allmydata.util.eliotutil import ( opt_eliot_destination, opt_help_eliot_destinations, @@ -112,6 +118,7 @@ def parse_options(argv, config=None): config.parseOptions(argv) # may raise usage.error return config + def parse_or_exit_with_explanation(argv, stdout=sys.stdout, stderr=sys.stderr, stdin=sys.stdin): config = Options() config.stdout = stdout @@ -124,7 +131,10 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout, stderr=sys.stderr, s while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - print("%s: %s\n" % (sys.argv[0], e), file=stdout) + # On Python 2 the string may turn into a unicode string, e.g. the error + # may be unicode, in which case it will print funny. Once we're on + # Python 3 we can just drop the ensure_str(). + print(six.ensure_str("%s: %s\n" % (sys.argv[0], e)), file=stdout) sys.exit(1) return config @@ -177,10 +187,12 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options +PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.") + def run(): if six.PY3: - warnings.warn("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.") + print(PYTHON_3_WARNING, file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize @@ -196,7 +208,8 @@ def _setup_coverage(reactor): """ # can we put this _setup_coverage call after we hit # argument-parsing? - if '--coverage' not in sys.argv: + # ensure_str() only necessary on Python 2. + if six.ensure_str('--coverage') not in sys.argv: return sys.argv.remove('--coverage') @@ -233,7 +246,8 @@ def _run_with_reactor(reactor): _setup_coverage(reactor) - d = defer.maybeDeferred(parse_or_exit_with_explanation, sys.argv[1:]) + argv = list(map(argv_to_unicode, sys.argv[1:])) + d = defer.maybeDeferred(parse_or_exit_with_explanation, argv) d.addCallback(_maybe_enable_eliot_logging, reactor) d.addCallback(dispatch) def _show_exception(f): diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index ce25e9667..3c23fb533 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,12 +1,24 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str + import os, time from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util import base32 from allmydata.util.encodingutil import quote_output, is_printable_ascii -import urllib +from urllib.parse import quote as url_quote import json class SlowOperationRunner(object): @@ -14,7 +26,7 @@ class SlowOperationRunner(object): def run(self, options): stderr = options.stderr self.options = options - self.ophandle = ophandle = base32.b2a(os.urandom(16)) + self.ophandle = ophandle = ensure_str(base32.b2a(os.urandom(16))) nodeurl = options['node-url'] if not nodeurl.endswith("/"): nodeurl += "/" @@ -25,9 +37,10 @@ class SlowOperationRunner(object): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -53,10 +66,10 @@ class SlowOperationRunner(object): def wait_for_results(self): last = 0 - for next in self.poll_times(): - delay = next - last + for next_item in self.poll_times(): + delay = next_item - last time.sleep(delay) - last = next + last = next_item if self.poll(): return 0 @@ -74,8 +87,13 @@ class SlowOperationRunner(object): if not data["finished"]: return False if self.options.get("raw"): + if PY3: + # need to write bytes! + stdout = stdout.buffer if is_printable_ascii(jdata): - print(jdata, file=stdout) + stdout.write(jdata) + stdout.write(b"\n") + stdout.flush() else: print("The JSON response contained unprintable characters:\n%s" % quote_output(jdata), file=stderr) return True diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 19474b9e8..8476aeb28 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -1,7 +1,14 @@ -from __future__ import print_function +""" +Ported to Python 3. +""" from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path import codecs @@ -37,7 +44,7 @@ def add_line_to_aliasfile(aliasfile, alias, cap): def add_alias(options): nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) cap = options.cap stdout = options.stdout stderr = options.stderr @@ -54,7 +61,7 @@ def add_alias(options): show_output(stderr, "Alias {alias} already exists!", alias=alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") - cap = unicode(uri.from_string_dirnode(cap).to_string(), 'utf-8') + cap = str(uri.from_string_dirnode(cap).to_string(), 'utf-8') add_line_to_aliasfile(aliasfile, alias, cap) show_output(stdout, "Alias {alias} added", alias=alias) @@ -64,7 +71,7 @@ def create_alias(options): # mkdir+add_alias nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) stdout = options.stdout stderr = options.stderr if u":" in alias: @@ -94,7 +101,7 @@ def create_alias(options): # probably check for others.. - add_line_to_aliasfile(aliasfile, alias, unicode(new_uri, "utf-8")) + add_line_to_aliasfile(aliasfile, alias, str(new_uri, "utf-8")) show_output(stdout, "Alias {alias} created", alias=alias) return 0 @@ -114,7 +121,7 @@ def show_output(fp, template, **kwargs): ``encoding`` attribute at all (eg StringIO.StringIO) by writing utf-8-encoded bytes. """ - assert isinstance(template, unicode) + assert isinstance(template, str) # On Python 3 fp has an encoding attribute under all real usage. On # Python 2, the encoding attribute is None if stdio is not a tty. The diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index c63558eb1..b574f16e8 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -1,14 +1,24 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path import time -import urllib -import json +from urllib.parse import quote as url_quote import datetime + from allmydata.scripts.common import get_alias, escape_path, DEFAULT_ALIAS, \ UnknownAliasError from allmydata.scripts.common_http import do_http, HTTPError, format_http_error -from allmydata.util import time_format +from allmydata.util import time_format, jsonbytes as json from allmydata.scripts import backupdb from allmydata.util.encodingutil import listdir_unicode, quote_output, \ quote_local_unicode_path, to_bytes, FilenameEncodingError, unicode_to_url @@ -20,7 +30,6 @@ def get_local_metadata(path): metadata = {} # posix stat(2) metadata, depends on the platform - os.stat_float_times(True) s = os.stat(path) metadata["ctime"] = s.st_ctime metadata["mtime"] = s.st_mtime @@ -52,7 +61,7 @@ def mkdir(contents, options): def put_child(dirurl, childname, childcap): assert dirurl[-1] != "/" - url = dirurl + "/" + urllib.quote(unicode_to_url(childname)) + "?t=uri" + url = dirurl + "/" + url_quote(unicode_to_url(childname)) + "?t=uri" resp = do_http("PUT", url, childcap) if resp.status not in (200, 201): raise HTTPError("Error during put_child", resp) @@ -97,7 +106,7 @@ class BackerUpper(object): except UnknownAliasError as e: e.display(stderr) return 1 - to_url = nodeurl + "uri/%s/" % urllib.quote(rootcap) + to_url = nodeurl + "uri/%s/" % url_quote(rootcap) if path: to_url += escape_path(path) if not to_url.endswith("/"): @@ -165,7 +174,7 @@ class BackerUpper(object): if must_create: self.verboseprint(" creating directory for %s" % quote_local_unicode_path(path)) newdircap = mkdir(create_contents, self.options) - assert isinstance(newdircap, str) + assert isinstance(newdircap, bytes) if r: r.did_create(newdircap) return True, newdircap @@ -192,7 +201,7 @@ class BackerUpper(object): filecap = r.was_uploaded() self.verboseprint("checking %s" % quote_output(filecap)) nodeurl = self.options['node-url'] - checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(filecap) + checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(filecap) self._files_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: @@ -225,7 +234,7 @@ class BackerUpper(object): dircap = r.was_created() self.verboseprint("checking %s" % quote_output(dircap)) nodeurl = self.options['node-url'] - checkurl = nodeurl + "uri/%s?t=check&output=JSON" % urllib.quote(dircap) + checkurl = nodeurl + "uri/%s?t=check&output=JSON" % url_quote(dircap) self._directories_checked += 1 resp = do_http("POST", checkurl) if resp.status != 200: @@ -292,7 +301,7 @@ def collect_backup_targets(root, listdir, filter_children): yield FilenameUndecodableTarget(root, isdir=True) else: for child in filter_children(children): - assert isinstance(child, unicode), child + assert isinstance(child, str), child childpath = os.path.join(root, child) if os.path.islink(childpath): yield LinkTarget(childpath, isdir=False) @@ -345,7 +354,7 @@ class FileTarget(object): target = PermissionDeniedTarget(self._path, isdir=False) return target.backup(progress, upload_file, upload_directory) else: - assert isinstance(childcap, str) + assert isinstance(childcap, bytes) if created: return progress.created_file(self._path, childcap, metadata) return progress.reused_file(self._path, childcap, metadata) @@ -497,10 +506,10 @@ class BackupProgress(object): ) def _format_elapsed(self, elapsed): - seconds = elapsed.total_seconds() - hours = int(seconds / 3600) - minutes = int(seconds / 60 % 60) - seconds = int(seconds % 60) + seconds = int(elapsed.total_seconds()) + hours = seconds // 3600 + minutes = (seconds // 60) % 60 + seconds = seconds % 60 return "{}h {}m {}s".format( hours, minutes, @@ -525,12 +534,12 @@ class BackupProgress(object): return self, { os.path.basename(create_path): create_value for (create_path, create_value) - in self._create_contents.iteritems() + in list(self._create_contents.items()) if os.path.dirname(create_path) == dirpath }, { os.path.basename(compare_path): compare_value for (compare_path, compare_value) - in self._compare_contents.iteritems() + in list(self._compare_contents.items()) if os.path.dirname(compare_path) == dirpath } diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index cef9e32be..6bafe3d1a 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,19 +1,26 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib -import json - -# Python 2 compatibility from future.utils import PY2 if PY2: - from future.builtins import str # noqa: F401 + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str, ensure_text + +from urllib.parse import quote as url_quote +import json from twisted.protocols.basic import LineOnlyReceiver from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.encodingutil import quote_output, quote_path +from allmydata.util.encodingutil import quote_output, quote_path, get_io_encoding class Checker(object): pass @@ -34,9 +41,10 @@ def check_location(options, where): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -52,7 +60,8 @@ def check_location(options, where): if resp.status != 200: print(format_http_error("ERROR", resp), file=stderr) return 1 - jdata = resp.read() + jdata = resp.read().decode() + if options.get("raw"): stdout.write(jdata) stdout.write("\n") @@ -139,7 +148,7 @@ class DeepCheckOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -166,7 +175,9 @@ class DeepCheckOutput(LineOnlyReceiver, object): # LIT files and directories do not have a "summary" field. summary = cr.get("summary", "Healthy (LIT)") - print("%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False)), file=stdout) + # When Python 2 is dropped the ensure_text()/ensure_str() will be unnecessary. + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), quote_output(summary, quotemarks=False)), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares for shareloc in cr["results"].get("list-corrupt-shares", []): @@ -202,7 +213,7 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -243,11 +254,14 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if not path: path = [""] # we don't seem to have a summary available, so build one + # When Python 2 is dropped the ensure_text/ensure_str crap can be + # dropped. if was_healthy: - summary = "healthy" + summary = ensure_str("healthy") else: - summary = "not healthy" - print("%s: %s" % (quote_path(path), summary), file=stdout) + summary = ensure_str("not healthy") + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), summary), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares prr = crr.get("pre-repair-results", {}) @@ -295,9 +309,10 @@ class DeepCheckStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -322,7 +337,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object): if not chunk: break if self.options["raw"]: - stdout.write(chunk) + stdout.write(chunk.decode()) else: output.dataReceived(chunk) if not self.options["raw"]: diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py index f7879f35c..aae03291f 100644 --- a/src/allmydata/scripts/tahoe_cp.py +++ b/src/allmydata/scripts/tahoe_cp.py @@ -1,10 +1,20 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path -import urllib -import json +from urllib.parse import quote as url_quote from collections import defaultdict -from six.moves import cStringIO as StringIO +from io import BytesIO + from twisted.python.failure import Failure from allmydata.scripts.common import get_alias, escape_path, \ DefaultAliasMarker, TahoeError @@ -15,6 +25,7 @@ from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abs from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \ quote_local_unicode_path, to_bytes from allmydata.util.assertutil import precondition, _assert +from allmydata.util import jsonbytes as json class MissingSourceError(TahoeError): @@ -61,8 +72,8 @@ def mkdir(targeturl): def make_tahoe_subdirectory(nodeurl, parent_writecap, name): url = nodeurl + "/".join(["uri", - urllib.quote(parent_writecap), - urllib.quote(unicode_to_url(name)), + url_quote(parent_writecap), + url_quote(unicode_to_url(name)), ]) + "?t=mkdir" resp = do_http("POST", url) if resp.status in (200, 201): @@ -159,7 +170,7 @@ class LocalDirectoryTarget(object): self.children[n] = LocalFileTarget(pn) def get_child_target(self, name): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) precondition(len(name), name) # don't want "" if self.children is None: self.populate(recurse=False) @@ -172,7 +183,7 @@ class LocalDirectoryTarget(object): return child def put_file(self, name, inf): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) pathname = os.path.join(self.pathname, name) fileutil.put_file(pathname, inf) @@ -198,13 +209,21 @@ class TahoeFileSource(object): def open(self, caps_only): if caps_only: - return StringIO(self.readcap) - url = self.nodeurl + "uri/" + urllib.quote(self.readcap) + return BytesIO(self.readcap) + url = self.nodeurl + "uri/" + url_quote(self.readcap) return GET_to_file(url) def bestcap(self): return self.writecap or self.readcap + +def seekable(file_like): + """Return whether the file-like object is seekable.""" + return hasattr(file_like, "seek") and ( + not hasattr(file_like, "seekable") or file_like.seekable() + ) + + class TahoeFileTarget(object): def __init__(self, nodeurl, mutable, writecap, readcap, url): self.nodeurl = nodeurl @@ -218,7 +237,7 @@ class TahoeFileTarget(object): assert self.url # our do_http() call currently requires a string or a filehandle with # a real .seek - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() PUT(self.url, inf) # TODO: this always creates immutable files. We might want an option @@ -239,7 +258,7 @@ class TahoeDirectorySource(object): self.writecap = writecap self.readcap = readcap bestcap = writecap or readcap - url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) + url = self.nodeurl + "uri/%s" % url_quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: raise HTTPError("Error examining source directory", resp) @@ -247,9 +266,9 @@ class TahoeDirectorySource(object): nodetype, d = parsed assert nodetype == "dirnode" self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def init_from_parsed(self, parsed): @@ -257,9 +276,9 @@ class TahoeDirectorySource(object): self.writecap = to_bytes(d.get("rw_uri")) self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def populate(self, recurse): @@ -304,7 +323,7 @@ class TahoeMissingTarget(object): def put_file(self, inf): # We want to replace this object in-place. - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() PUT(self.url, inf) # TODO: this always creates immutable files. We might want an option @@ -327,16 +346,16 @@ class TahoeDirectoryTarget(object): self.writecap = to_bytes(d.get("rw_uri")) self.readcap = to_bytes(d.get("ro_uri")) self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def init_from_grid(self, writecap, readcap): self.writecap = writecap self.readcap = readcap bestcap = writecap or readcap - url = self.nodeurl + "uri/%s" % urllib.quote(bestcap) + url = self.nodeurl + "uri/%s" % url_quote(bestcap) resp = do_http("GET", url + "?t=json") if resp.status != 200: raise HTTPError("Error examining target directory", resp) @@ -344,9 +363,9 @@ class TahoeDirectoryTarget(object): nodetype, d = parsed assert nodetype == "dirnode" self.mutable = d.get("mutable", False) # older nodes don't provide it - self.children_d = dict( [(unicode(name),value) + self.children_d = dict( [(str(name),value) for (name,value) - in d["children"].iteritems()] ) + in d["children"].items()] ) self.children = None def just_created(self, writecap): @@ -370,8 +389,8 @@ class TahoeDirectoryTarget(object): url = None if self.writecap: url = self.nodeurl + "/".join(["uri", - urllib.quote(self.writecap), - urllib.quote(unicode_to_url(name))]) + url_quote(self.writecap), + url_quote(unicode_to_url(name))]) self.children[name] = TahoeFileTarget(self.nodeurl, mutable, writecap, readcap, url) elif data[0] == "dirnode": @@ -400,7 +419,7 @@ class TahoeDirectoryTarget(object): def get_child_target(self, name): # return a new target for a named subdirectory of this dir - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) if self.children is None: self.populate(recurse=False) if name in self.children: @@ -413,9 +432,9 @@ class TahoeDirectoryTarget(object): return child def put_file(self, name, inf): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) url = self.nodeurl + "uri" - if not hasattr(inf, "seek"): + if not seekable(inf): inf = inf.read() if self.children is None: @@ -433,16 +452,16 @@ class TahoeDirectoryTarget(object): self.new_children[name] = filecap def put_uri(self, name, filecap): - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) self.new_children[name] = filecap def set_children(self): if not self.new_children: return - url = (self.nodeurl + "uri/" + urllib.quote(self.writecap) + url = (self.nodeurl + "uri/" + url_quote(self.writecap) + "?t=set_children") set_data = {} - for (name, filecap) in self.new_children.items(): + for (name, filecap) in list(self.new_children.items()): # it just so happens that ?t=set_children will accept both file # read-caps and write-caps as ['rw_uri'], and will handle either # correctly. So don't bother trying to figure out whether the one @@ -450,7 +469,7 @@ class TahoeDirectoryTarget(object): # TODO: think about how this affects forward-compatibility for # unknown caps set_data[name] = ["filenode", {"rw_uri": filecap}] - body = json.dumps(set_data) + body = json.dumps_bytes(set_data) POST(url, body) FileSources = (LocalFileSource, TahoeFileSource) @@ -586,7 +605,7 @@ class Copier(object): # and get_source_info. def get_target_info(self, destination_spec): - precondition(isinstance(destination_spec, unicode), destination_spec) + precondition(isinstance(destination_spec, str), destination_spec) rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None) path = path_utf8.decode("utf-8") if rootcap == DefaultAliasMarker: @@ -603,7 +622,7 @@ class Copier(object): t = LocalFileTarget(pathname) # non-empty else: # this is a tahoe object - url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) + url = self.nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) @@ -633,7 +652,7 @@ class Copier(object): """ This turns an argv string into a (Local|Tahoe)(File|Directory)Source. """ - precondition(isinstance(source_spec, unicode), source_spec) + precondition(isinstance(source_spec, str), source_spec) rootcap, path_utf8 = get_alias(self.aliases, source_spec, None) path = path_utf8.decode("utf-8") # any trailing slash is removed in abspath_expanduser_unicode(), so @@ -656,7 +675,7 @@ class Copier(object): t = LocalFileSource(pathname, name) # non-empty else: # this is a tahoe object - url = self.nodeurl + "uri/%s" % urllib.quote(rootcap) + url = self.nodeurl + "uri/%s" % url_quote(rootcap) name = None if path: if path.endswith("/"): @@ -690,6 +709,8 @@ class Copier(object): def need_to_copy_bytes(self, source, target): + # This should likley be a method call! but enabling that triggers + # additional bugs. https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3719 if source.need_to_copy_bytes: # mutable tahoe files, and local files return True @@ -739,7 +760,7 @@ class Copier(object): # target name collisions are an error collisions = [] - for target, sources in targetmap.items(): + for target, sources in list(targetmap.items()): target_names = {} for source in sources: name = source.basename() @@ -808,7 +829,7 @@ class Copier(object): def assign_targets(self, targetmap, source, target): # copy everything in the source into the target precondition(isinstance(source, DirectorySources), source) - for name, child in source.children.items(): + for name, child in list(source.children.items()): if isinstance(child, DirectorySources): # we will need a target directory for this one subtarget = target.get_child_target(name) @@ -824,7 +845,7 @@ class Copier(object): files_copied = 0 targets_finished = 0 - for target, sources in targetmap.items(): + for target, sources in list(targetmap.items()): _assert(isinstance(target, DirectoryTargets), target) for source in sources: _assert(isinstance(source, FileSources), source) @@ -844,7 +865,7 @@ class Copier(object): def copy_file_into_dir(self, source, name, target): precondition(isinstance(source, FileSources), source) precondition(isinstance(target, DirectoryTargets), target) - precondition(isinstance(name, unicode), name) + precondition(isinstance(name, str), name) if self.need_to_copy_bytes(source, target): # if the target is a local directory, this will just write the # bytes to disk. If it is a tahoe directory, it will upload the diff --git a/src/allmydata/scripts/tahoe_get.py b/src/allmydata/scripts/tahoe_get.py index d90baf2c9..39f1686ce 100644 --- a/src/allmydata/scripts/tahoe_get.py +++ b/src/allmydata/scripts/tahoe_get.py @@ -1,6 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error @@ -20,7 +30,7 @@ def get(options): except UnknownAliasError as e: e.display(stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) @@ -30,6 +40,10 @@ def get(options): outf = open(to_file, "wb") else: outf = stdout + # Make sure we can write bytes; on Python 3 stdout is Unicode by + # default. + if PY3 and getattr(outf, "encoding", None) is not None: + outf = outf.buffer while True: data = resp.read(4096) if not data: diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 884536ec2..09d4cbd59 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import json +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 try: from allmydata.scripts.types_ import SubCommands @@ -13,8 +21,9 @@ from twisted.internet import defer, reactor from wormhole import wormhole from allmydata.util.encodingutil import argv_to_abspath +from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl -from allmydata.node import read_config +from allmydata.client import read_config class InviteOptions(usage.Options): @@ -54,7 +63,7 @@ def _send_config_via_wormhole(options, config): code = yield wh.get_code() print("Invite Code for client: {}".format(code), file=out) - wh.send_message(json.dumps({ + wh.send_message(json.dumps_bytes({ u"abilities": { u"server-v1": {}, } @@ -71,7 +80,7 @@ def _send_config_via_wormhole(options, config): defer.returnValue(1) print(" transmitting configuration", file=out) - wh.send_message(json.dumps(config)) + wh.send_message(json.dumps_bytes(config)) yield wh.close() @@ -94,9 +103,9 @@ def invite(options): nick = options['nick'] remote_config = { - "shares-needed": options["shares-needed"] or config.get('client', 'shares.needed'), - "shares-total": options["shares-total"] or config.get('client', 'shares.total'), - "shares-happy": options["shares-happy"] or config.get('client', 'shares.happy'), + "shares-needed": options["shares-needed"] or config.get_config('client', 'shares.needed'), + "shares-total": options["shares-total"] or config.get_config('client', 'shares.total'), + "shares-happy": options["shares-happy"] or config.get_config('client', 'shares.happy'), "nickname": nick, "introducer": introducer_furl, } diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 2bfe16d27..5a7136d77 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -1,13 +1,26 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib, time +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_text + +import time +from urllib.parse import quote as url_quote import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util.encodingutil import unicode_to_output, quote_output, is_printable_ascii, to_bytes -def list(options): +def ls(options): nodeurl = options['node-url'] aliases = options.aliases where = options.where @@ -23,7 +36,9 @@ def list(options): except UnknownAliasError as e: e.display(stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + + path = str(path, "utf-8") + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: # move where.endswith check here? url += "/" + escape_path(path) @@ -41,10 +56,10 @@ def list(options): return resp.status data = resp.read() - if options['json']: # The webapi server should always output printable ASCII. if is_printable_ascii(data): + data = str(data, "ascii") print(data, file=stdout) return 0 else: @@ -66,7 +81,7 @@ def list(options): children = d['children'] else: # paths returned from get_alias are always valid UTF-8 - childname = path.split("/")[-1].decode('utf-8') + childname = path.split("/")[-1] children = {childname: (nodetype, d)} if "metadata" not in d: d["metadata"] = {} @@ -81,7 +96,7 @@ def list(options): for name in childnames: child = children[name] - name = unicode(name) + name = str(name) childtype = child[0] # See webapi.txt for a discussion of the meanings of unix local @@ -141,30 +156,25 @@ def list(options): if not options["classify"]: classify = "" - encoding_error = False - try: - line.append(unicode_to_output(name) + classify) - except UnicodeEncodeError: - encoding_error = True - line.append(quote_output(name) + classify) + line.append(name + classify) if options["uri"]: - line.append(uri) + line.append(ensure_text(uri)) if options["readonly-uri"]: - line.append(quote_output(ro_uri or "-", quotemarks=False)) + line.append(quote_output(ensure_text(ro_uri) or "-", quotemarks=False)) - rows.append((encoding_error, line)) + rows.append(line) max_widths = [] left_justifys = [] - for (encoding_error, row) in rows: + for row in rows: for i,cell in enumerate(row): while len(max_widths) <= i: max_widths.append(0) while len(left_justifys) <= i: left_justifys.append(False) max_widths[i] = max(max_widths[i], len(cell)) - if cell.startswith("URI"): + if ensure_text(cell).startswith("URI"): left_justifys[i] = True if len(left_justifys) == 1: left_justifys[0] = True @@ -179,12 +189,19 @@ def list(options): fmt = " ".join(fmt_pieces) rc = 0 - for (encoding_error, row) in rows: + for row in rows: + row = (fmt % tuple(row)).rstrip() + encoding_error = False + try: + row = unicode_to_output(row) + except UnicodeEncodeError: + encoding_error = True + row = quote_output(row) if encoding_error: - print((fmt % tuple(row)).rstrip(), file=stderr) + print(row, file=stderr) rc = 1 else: - print((fmt % tuple(row)).rstrip(), file=stdout) + print(row, file=stdout) if rc == 1: print("\nThis listing included files whose names could not be converted to the terminal" \ diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 386cdd1ad..b55075eef 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,6 +1,19 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib, json +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str + +from urllib.parse import quote as url_quote +import json from twisted.protocols.basic import LineOnlyReceiver from allmydata.util.abbreviate import abbreviate_space_both from allmydata.scripts.slow_operation import SlowOperationRunner @@ -33,9 +46,10 @@ class ManifestStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -47,6 +61,9 @@ class ManifestStreamer(LineOnlyReceiver, object): #print("RESP", dir(resp)) # use Twisted to split this into lines self.in_error = False + # Writing bytes, so need binary stdout. + if PY3: + stdout = stdout.buffer while True: chunk = resp.read(100) if not chunk: @@ -63,7 +80,7 @@ class ManifestStreamer(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.rc = 1 print(quote_output(line, quotemarks=False), file=stderr) @@ -88,8 +105,10 @@ class ManifestStreamer(LineOnlyReceiver, object): if vc: print(quote_output(vc, quotemarks=False), file=stdout) else: - print("%s %s" % (quote_output(d["cap"], quotemarks=False), - quote_path(d["path"], quotemarks=False)), file=stdout) + # ensure_str() only necessary for Python 2. + print(ensure_str("%s %s") % ( + quote_output(d["cap"], quotemarks=False), + quote_path(d["path"], quotemarks=False)), file=stdout) def manifest(options): return ManifestStreamer().run(options) diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py index a76adc8fc..85fe12554 100644 --- a/src/allmydata/scripts/tahoe_mkdir.py +++ b/src/allmydata/scripts/tahoe_mkdir.py @@ -1,6 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError from allmydata.util.encodingutil import quote_output @@ -24,7 +34,7 @@ def mkdir(options): # create a new unlinked directory url = nodeurl + "uri?t=mkdir" if options["format"]: - url += "&format=%s" % urllib.quote(options['format']) + url += "&format=%s" % url_quote(options['format']) resp = do_http("POST", url) rc = check_http_error(resp, stderr) if rc: @@ -35,13 +45,14 @@ def mkdir(options): return 0 # create a new directory at the given location + path = str(path, "utf-8") if path.endswith("/"): path = path[:-1] # path must be "/".join([s.encode("utf-8") for s in segments]) - url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap), - urllib.quote(path)) + url = nodeurl + "uri/%s/%s?t=mkdir" % (url_quote(rootcap), + url_quote(path)) if options['format']: - url += "&format=%s" % urllib.quote(options['format']) + url += "&format=%s" % url_quote(options['format']) resp = do_http("POST", url) check_http_error(resp, stderr) diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index 7d13ea72a..d921047a8 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,7 +1,17 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import re -import urllib +from urllib.parse import quote as url_quote import json from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -25,7 +35,8 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 - from_url = nodeurl + "uri/%s" % urllib.quote(rootcap) + from_path = str(from_path, "utf-8") + from_url = nodeurl + "uri/%s" % url_quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) # figure out the source cap @@ -43,7 +54,8 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 - to_url = nodeurl + "uri/%s" % urllib.quote(rootcap) + to_url = nodeurl + "uri/%s" % url_quote(rootcap) + path = str(path, "utf-8") if path: to_url += "/" + escape_path(path) diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py index 8d87408dc..1ea45e8ea 100644 --- a/src/allmydata/scripts/tahoe_put.py +++ b/src/allmydata/scripts/tahoe_put.py @@ -1,7 +1,17 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from six.moves import cStringIO as StringIO -import urllib +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from io import BytesIO +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ @@ -46,19 +56,20 @@ def put(options): # FIXME: don't hardcode cap format. if to_file.startswith("URI:MDMF:") or to_file.startswith("URI:SSK:"): - url = nodeurl + "uri/%s" % urllib.quote(to_file) + url = nodeurl + "uri/%s" % url_quote(to_file) else: try: rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS) except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path.startswith("/"): suggestion = to_file.replace(u"/", u"", 1) print("Error: The remote filename must not start with a slash", file=stderr) print("Please try again, perhaps with %s" % quote_output(suggestion), file=stderr) return 1 - url = nodeurl + "uri/%s/" % urllib.quote(rootcap) + url = nodeurl + "uri/%s/" % url_quote(rootcap) if path: url += escape_path(path) else: @@ -80,8 +91,13 @@ def put(options): # Content-Length field. So we currently must copy it. if verbosity > 0: print("waiting for file data on stdin..", file=stderr) - data = stdin.read() - infileobj = StringIO(data) + # We're uploading arbitrary files, so this had better be bytes: + if PY2: + stdinb = stdin + else: + stdinb = stdin.buffer + data = stdinb.read() + infileobj = BytesIO(data) resp = do_http("PUT", url, infileobj) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index bc4ba27d1..218d8d458 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + __all__ = [ "RunOptions", "run", diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index ff746901b..75bc775f8 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.builtins import chr +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from urllib.parse import urlencode, quote as url_quote @@ -53,7 +61,7 @@ def _get_json_for_cap(options, cap): 'uri/%s?t=json' % url_quote(cap), ) -def pretty_progress(percent, size=10, ascii=False): +def pretty_progress(percent, size=10, output_ascii=False): """ Displays a unicode or ascii based progress bar of a certain length. Should we just depend on a library instead? @@ -64,7 +72,7 @@ def pretty_progress(percent, size=10, ascii=False): curr = int(percent / 100.0 * size) part = (percent / (100.0 / size)) - curr - if ascii: + if output_ascii: part = int(part * 4) part = '.oO%'[part] block_chr = '#' diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py index bc1d43c9e..5bdebb960 100644 --- a/src/allmydata/scripts/tahoe_unlink.py +++ b/src/allmydata/scripts/tahoe_unlink.py @@ -1,6 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -import urllib +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from urllib.parse import quote as url_quote from allmydata.scripts.common_http import do_http, format_http_success, format_http_error from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError @@ -27,7 +37,7 @@ def unlink(options, command="unlink"): 'tahoe %s' can only unlink directory entries, so a path must be given.""" % (command,), file=stderr) return 1 - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) url += "/" + escape_path(path) resp = do_http("DELETE", url) diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py index a7b7ca7e1..dbec31e87 100644 --- a/src/allmydata/scripts/tahoe_webopen.py +++ b/src/allmydata/scripts/tahoe_webopen.py @@ -1,7 +1,20 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from urllib.parse import quote as url_quote from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError -import urllib + def webopen(options, opener=None): nodeurl = options['node-url'] @@ -15,9 +28,10 @@ def webopen(options, opener=None): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) else: diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 7137ba28e..13ed8817c 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -9,7 +9,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - + from time import clock as process_time +else: + from time import process_time import time from twisted.application import service @@ -27,7 +29,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - # we don't use time.clock() here, because the constructor is run by + # we don't use process_time() here, because the constructor is run by # the twistd parent process (as it loads the .tac file), whereas the # rest of the program will be run by the child process, after twistd # forks. Instead, set self.initial_cpu as soon as the reactor starts @@ -39,11 +41,11 @@ class CPUUsageMonitor(service.MultiService): TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) def _set_initial_cpu(self): - self.initial_cpu = time.clock() + self.initial_cpu = process_time() def check(self): now_wall = time.time() - now_cpu = time.clock() + now_cpu = process_time() self.samples.append( (now_wall, now_cpu) ) while len(self.samples) > self.HISTORY_LENGTH+1: self.samples.pop(0) @@ -68,7 +70,7 @@ class CPUUsageMonitor(service.MultiService): avg = self._average_N_minutes(15) if avg is not None: s["cpu_monitor.15min_avg"] = avg - now_cpu = time.clock() + now_cpu = process_time() s["cpu_monitor.total"] = now_cpu - self.initial_cpu return s diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index f13f7cb99..bd4f4f432 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -252,7 +252,9 @@ class ShareCrawler(service.MultiService): self.state["last-complete-prefix"] = last_complete_prefix tmpfile = self.statefile + ".tmp" with open(tmpfile, "wb") as f: - pickle.dump(self.state, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(self.state, f, protocol=2) fileutil.move_into_place(tmpfile, self.statefile) def startService(self): diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py index ffe2bf774..7c6cd8218 100644 --- a/src/allmydata/storage/expirer.py +++ b/src/allmydata/storage/expirer.py @@ -95,7 +95,9 @@ class LeaseCheckingCrawler(ShareCrawler): if not os.path.exists(self.historyfile): history = {} # cyclenum -> dict with open(self.historyfile, "wb") as f: - pickle.dump(history, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(history, f, protocol=2) def create_empty_cycle_dict(self): recovered = self.create_empty_recovered_dict() @@ -319,7 +321,9 @@ class LeaseCheckingCrawler(ShareCrawler): oldcycles = sorted(history.keys()) del history[oldcycles[0]] with open(self.historyfile, "wb") as f: - pickle.dump(history, f) + # Newer protocols won't work in Python 2; when it is dropped, + # protocol v4 can be used (added in Python 3.4). + pickle.dump(history, f, protocol=2) def get_state(self): """In addition to the crawler state described in diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index e3ac48290..893aa15ce 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -12,11 +12,18 @@ Some setup that should apply across the entire test suite. Rather than defining interesting APIs for other code to use, this just causes some side-effects which make things better when the test suite runs. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from future.utils import PY3 +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -import warnings from traceback import extract_stack, format_list from foolscap.pb import Listener @@ -25,11 +32,6 @@ from twisted.application import service from foolscap.logging.incident import IncidentQualifier -if PY3: - # Error on BytesWarnings, to catch things like str(b""), but only for - # allmydata code. - warnings.filterwarnings("error", category=BytesWarning, module="allmydata.*") - class NonQualifier(IncidentQualifier, object): def check_event(self, ev): @@ -123,5 +125,5 @@ if sys.platform == "win32": initialize() from eliot import to_file -from allmydata.util.jsonbytes import BytesJSONEncoder -to_file(open("eliot.log", "w"), encoder=BytesJSONEncoder) +from allmydata.util.jsonbytes import AnyBytesJSONEncoder +to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder) diff --git a/src/allmydata/test/_twisted_9607.py b/src/allmydata/test/_twisted_9607.py deleted file mode 100644 index c4e37ef38..000000000 --- a/src/allmydata/test/_twisted_9607.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -A copy of the implementation of Twisted's ``getProcessOutputAndValue`` -with the fix for Twisted #9607 (support for stdinBytes) patched in. -""" - -from __future__ import ( - division, - absolute_import, - print_function, - unicode_literals, -) - -from io import BytesIO - -from twisted.internet import protocol, defer - - -class _EverythingGetter(protocol.ProcessProtocol, object): - - def __init__(self, deferred, stdinBytes=None): - self.deferred = deferred - self.outBuf = BytesIO() - self.errBuf = BytesIO() - self.outReceived = self.outBuf.write - self.errReceived = self.errBuf.write - self.stdinBytes = stdinBytes - - def connectionMade(self): - if self.stdinBytes is not None: - self.transport.writeToChild(0, self.stdinBytes) - # The only compelling reason not to _always_ close stdin here is - # backwards compatibility. - self.transport.closeStdin() - - def processEnded(self, reason): - out = self.outBuf.getvalue() - err = self.errBuf.getvalue() - e = reason.value - code = e.exitCode - if e.signal: - self.deferred.errback((out, err, e.signal)) - else: - self.deferred.callback((out, err, code)) - - - -def _callProtocolWithDeferred(protocol, executable, args, env, path, - reactor=None, protoArgs=()): - if reactor is None: - from twisted.internet import reactor - - d = defer.Deferred() - p = protocol(d, *protoArgs) - reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) - return d - - - -def getProcessOutputAndValue(executable, args=(), env={}, path=None, - reactor=None, stdinBytes=None): - """Spawn a process and returns a Deferred that will be called back with - its output (from stdout and stderr) and it's exit code as (out, err, code) - If a signal is raised, the Deferred will errback with the stdout and - stderr up to that point, along with the signal, as (out, err, signalNum) - """ - return _callProtocolWithDeferred( - _EverythingGetter, - executable, - args, - env, - path, - reactor, - protoArgs=(stdinBytes,), - ) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py index fe6960c73..bf9767e73 100644 --- a/src/allmydata/test/_win_subprocess.py +++ b/src/allmydata/test/_win_subprocess.py @@ -1,3 +1,16 @@ +""" +This module is only necessary on Python 2. Once Python 2 code is dropped, it +can be deleted. +""" + +from future.utils import PY3 +if PY3: + raise RuntimeError("Just use subprocess.Popen") + +# This is necessary to pacify flake8 on Python 3, while we're still supporting +# Python 2. +from past.builtins import unicode + # -*- coding: utf-8 -*- ## Copyright (C) 2021 Valentin Lab diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index 8796f815f..ed066c6b6 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -1,9 +1,22 @@ -from six import ensure_str +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str, ensure_text from ...scripts import runner from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode def parse_options(basedir, command, args): + args = [ensure_text(s) for s in args] o = runner.Options() o.parseOptions(["--node-directory", basedir, command] + args) while hasattr(o, "subOptions"): diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index ceecbd662..df598b811 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path from six.moves import cStringIO as StringIO from datetime import timedelta @@ -6,7 +18,6 @@ import re from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher -import __builtin__ from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv @@ -86,7 +97,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: do_backup(True)) def _check0(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) ( files_uploaded, @@ -143,40 +154,40 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "--uri", "tahoe:backups")) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.split("\n") children = dict([line.split() for line in lines if line]) latest_uri = children["Latest"] self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri) - childnames = children.keys() + childnames = list(children.keys()) self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"]) d.addCallback(_check1) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest")) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(sorted(out.split()), ["empty", "parent"]) d.addCallback(_check2) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Latest/empty")) def _check2a(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.strip(), "") + self.assertFalse(out.strip()) d.addCallback(_check2a) d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "foo") + self.assertEqual(out, "foo") d.addCallback(_check3) d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.old_archives = out.split() self.failUnlessReallyEqual(len(self.old_archives), 1) @@ -189,7 +200,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # second backup should reuse everything, if the backupdb is # available (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) # foo.txt, bar.txt, blah.txt @@ -221,7 +232,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # the directories should have been changed, so we should # re-use all of them too. (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) fchecked, dchecked = self.count_output2(out) @@ -238,7 +249,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() self.failUnlessReallyEqual(len(self.new_archives), 3, out) @@ -265,7 +276,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): # second backup should reuse bar.txt (if backupdb is available), # and upload the rest. None of the directories can be reused. (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) fu, fr, fs, dc, dr, ds = self.count_output(out) # new foo.txt, surprise file, subfile, empty @@ -281,7 +292,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("ls", "tahoe:backups/Archives")) def _check6(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) self.new_archives = out.split() self.failUnlessReallyEqual(len(self.new_archives), 4) @@ -291,17 +302,17 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("get", "tahoe:backups/Latest/parent/subdir/foo.txt")) def _check7(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "FOOF!") + self.assertEqual(out, "FOOF!") # the old snapshot should not be modified return self.do_cli("get", "tahoe:backups/Archives/%s/parent/subdir/foo.txt" % self.old_archives[0]) d.addCallback(_check7) def _check8(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertFalse(err) self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out, "foo") + self.assertEqual(out, "foo") d.addCallback(_check8) return d @@ -342,14 +353,14 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): exclusion_string = "_darcs\n*py\n.svn" excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(subdir_listdir)) self._check_filtering(filtered, subdir_listdir, (u'another_doc.lyx', u'CVS'), (u'.svn', u'_darcs', u'run_snake_run.py')) # test BackupConfigurationError self.failUnlessRaises(cli.BackupConfigurationError, parse, - ['--exclude-from', excl_filepath + '.no', 'from', 'to']) + ['--exclude-from-utf-8', excl_filepath + '.no', 'from', 'to']) # test that an iterator works too backup_options = parse(['--exclude', '*lyx', 'from', 'to']) @@ -360,7 +371,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding()) + doc_pattern_arg_unicode = doc_pattern_arg = u"*d\u00F8c*" + if PY2: + doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding()) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") @@ -382,10 +395,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) # read exclude patterns from file - exclusion_string = doc_pattern_arg + "\nlib.?" + exclusion_string = (doc_pattern_arg_unicode + "\nlib.?").encode("utf-8") excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) @@ -407,13 +420,21 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): ns = Namespace() ns.called = False - def call_file(name, *args): - ns.called = True - self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) - return StringIO() + original_open = open + def call_file(name, *args, **kwargs): + if name.endswith("excludes.dummy"): + ns.called = True + self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) + return StringIO() + else: + return original_open(name, *args, **kwargs) - patcher = MonkeyPatcher((__builtin__, 'file', call_file)) - patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) + if PY2: + from allmydata.scripts import cli as module_to_patch + else: + import builtins as module_to_patch + patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) + patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) def test_ignore_symlinks(self): @@ -584,7 +605,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0) d.addCallback(_check) return d @@ -600,6 +621,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0) d.addCallback(_check) return d diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 8cf963da6..472105ca1 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,3 +1,13 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text + import os.path import json from twisted.trial import unittest @@ -5,20 +15,21 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import quote_output, to_bytes +from allmydata.util.encodingutil import to_bytes, quote_output_u from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug from ..no_network import GridTestMixin from .common import CLITestMixin + class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def test_check(self): self.basedir = "cli/Check/check" self.set_up_grid() c0 = self.g.clients[0] - DATA = "data" * 100 + DATA = b"data" * 100 DATA_uploadable = MutableData(DATA) d = c0.create_mutable_file(DATA_uploadable) def _stash_uri(n): @@ -28,7 +39,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.uri)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy" in lines, out) @@ -38,14 +49,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) - self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy") + self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) - d.addCallback(lambda ign: c0.upload(upload.Data("literal", convergence=""))) + d.addCallback(lambda ign: c0.upload(upload.Data(b"literal", convergence=b""))) def _stash_lit_uri(n): self.lit_uri = n.get_uri() d.addCallback(_stash_lit_uri) @@ -53,7 +64,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.lit_uri)) def _check_lit(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy (LIT)" in lines, out) @@ -62,13 +73,13 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri)) def _check_lit_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check_lit_raw) - d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence="")) + d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=b"")) def _stash_lit_dir_uri(n): self.lit_dir_uri = n.get_uri() d.addCallback(_stash_lit_dir_uri) @@ -89,16 +100,16 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), - shares[1][0]) + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), + shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() summary = [l for l in lines if l.startswith("Summary")][0] @@ -112,7 +123,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", "--raw", self.uri)) def _check3_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], False) @@ -126,7 +137,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: not healthy" in lines, out) @@ -140,7 +151,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: healthy" in lines, out) @@ -156,14 +167,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 - quoted_good = quote_output(u"g\u00F6\u00F6d") + DATA = b"data" * 100 + quoted_good = quote_output_u("g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence="")) + return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -171,18 +182,18 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_uri, u"g\u00F6\u00F6d") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", - convergence=""))) + upload.Data(b"literal", + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: - c0.create_mutable_file(MutableData(DATA+"1"))) + c0.create_mutable_file(MutableData(DATA+b"1"))) d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn)) d.addCallback(_stash_uri, "mutable") d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy" @@ -198,8 +209,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -212,7 +224,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 1", lines) @@ -236,8 +248,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -251,8 +263,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -268,8 +281,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -287,7 +301,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() units = [json.loads(line) for line in lines] @@ -301,8 +315,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check6(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': healthy" in lines, out) self.failUnless("'small': healthy" in lines, out) @@ -322,10 +337,10 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir")) d.addCallback(_stash_uri, "subdir") d.addCallback(lambda fn: - fn.add_file(u"subfile", upload.Data(DATA+"2", ""))) + fn.add_file(u"subfile", upload.Data(DATA+b"2", b""))) d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], - range(10))) + list(range(10)))) # root # rootg\u00F6\u00F6d/ @@ -340,7 +355,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(rc, 0) self.failUnlessIn("ERROR: UnrecoverableFileError", err) # the fatal directory should still show up, as the last line - self.failUnlessIn(" subdir\n", out) + self.failUnlessIn(" subdir\n", ensure_text(out)) d.addCallback(_manifest_failed) d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) @@ -379,7 +394,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("deep-check")) d.addCallback(_check) @@ -396,7 +411,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -416,10 +431,10 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) #Ensure healthy appears for each uri - self.failUnlessIn("Healthy", out[:len(out)/2]) - self.failUnlessIn("Healthy", out[len(out)/2:]) + self.failUnlessIn("Healthy", out[:len(out)//2]) + self.failUnlessIn("Healthy", out[len(out)//2:]) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], "nonexistent:")) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2b1bc1c86..8a9b4dfd6 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,8 +1,23 @@ -import os.path +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six.moves import cStringIO as StringIO -import urllib, sys +from six import ensure_text, ensure_str + +import os.path +import sys import re from mock import patch, Mock +from urllib.parse import quote as url_quote from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher @@ -44,6 +59,7 @@ from allmydata.util.encodingutil import listdir_unicode, get_io_encoding class CLI(CLITestMixin, unittest.TestCase): def _dump_cap(self, *args): + args = [ensure_text(s) for s in args] config = debug.DumpCapOptions() config.stdout,config.stderr = StringIO(), StringIO() config.parseOptions(args) @@ -53,8 +69,8 @@ class CLI(CLITestMixin, unittest.TestCase): return output def test_dump_cap_chk(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - uri_extension_hash = hashutil.uri_extension_hash("stuff") + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -75,14 +91,14 @@ class CLI(CLITestMixin, unittest.TestCase): u.to_string()) self.failUnless("client renewal secret: znxmki5zdibb5qlt46xbdvk2t55j7hibejq3i5ijyurkr6m6jkhq" in output, output) - output = self._dump_cap(u.get_verify_cap().to_string()) + output = self._dump_cap(str(u.get_verify_cap().to_string(), "ascii")) self.failIf("key: " in output, output) self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output) self.failUnless("size: 1234" in output, output) self.failUnless("k/N: 25/100" in output, output) self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) - prefixed_u = "http://127.0.0.1/uri/%s" % urllib.quote(u.to_string()) + prefixed_u = "http://127.0.0.1/uri/%s" % url_quote(u.to_string()) output = self._dump_cap(prefixed_u) self.failUnless("CHK File:" in output, output) self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output) @@ -92,14 +108,14 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) def test_dump_cap_lit(self): - u = uri.LiteralFileURI("this is some data") + u = uri.LiteralFileURI(b"this is some data") output = self._dump_cap(u.to_string()) self.failUnless("Literal File URI:" in output, output) self.failUnless("data: 'this is some data'" in output, output) def test_dump_cap_sdmf(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u = uri.WriteableSSKFileURI(writekey, fingerprint) output = self._dump_cap(u.to_string()) @@ -149,8 +165,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output) def test_dump_cap_mdmf(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u = uri.WriteableMDMFFileURI(writekey, fingerprint) output = self._dump_cap(u.to_string()) @@ -201,8 +217,8 @@ class CLI(CLITestMixin, unittest.TestCase): def test_dump_cap_chk_directory(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - uri_extension_hash = hashutil.uri_extension_hash("stuff") + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -235,8 +251,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output) def test_dump_cap_sdmf_directory(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u1 = uri.WriteableSSKFileURI(writekey, fingerprint) u = uri.DirectoryURI(u1) @@ -279,8 +295,8 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output) def test_dump_cap_mdmf_directory(self): - writekey = "\x01" * 16 - fingerprint = "\xfe" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\xfe" * 32 u1 = uri.WriteableMDMFFileURI(writekey, fingerprint) u = uri.MDMFDirectoryURI(u1) @@ -340,7 +356,7 @@ class CLI(CLITestMixin, unittest.TestCase): fileutil.write("cli/test_catalog_shares/node1/storage/shares/mq/not-a-dir", "") # write a bogus share that looks a little bit like CHK fileutil.write(os.path.join(sharedir, "8"), - "\x00\x00\x00\x01" + "\xff" * 200) # this triggers an assert + b"\x00\x00\x00\x01" + b"\xff" * 200) # this triggers an assert nodedir2 = "cli/test_catalog_shares/node2" fileutil.make_dirs(nodedir2) @@ -348,7 +364,7 @@ class CLI(CLITestMixin, unittest.TestCase): # now make sure that the 'catalog-shares' commands survives the error out, err = self._catalog_shares(nodedir1, nodedir2) - self.failUnlessReallyEqual(out, "", out) + self.assertEqual(out, "") self.failUnless("Error processing " in err, "didn't see 'error processing' in '%s'" % err) #self.failUnless(nodedir1 in err, @@ -361,71 +377,71 @@ class CLI(CLITestMixin, unittest.TestCase): "didn't see 'mqfblse6m5a6dh45isu2cg7oji' in '%s'" % err) def test_alias(self): - def s128(c): return base32.b2a(c*(128/8)) - def s256(c): return base32.b2a(c*(256/8)) - TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T")) - WA = "URI:DIR2:%s:%s" % (s128("W"), s256("W")) - CA = "URI:DIR2:%s:%s" % (s128("C"), s256("C")) + def s128(c): return base32.b2a(c*(128//8)) + def s256(c): return base32.b2a(c*(256//8)) + TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T")) + WA = b"URI:DIR2:%s:%s" % (s128(b"W"), s256(b"W")) + CA = b"URI:DIR2:%s:%s" % (s128(b"C"), s256(b"C")) aliases = {"tahoe": TA, "work": WA, "c": CA} def ga1(path): return get_alias(aliases, path, u"tahoe") uses_lettercolon = common.platform_uses_lettercolon_drivename() - self.failUnlessReallyEqual(ga1(u"bare"), (TA, "bare")) - self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, "baredir/file")) - self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, "baredir/file:7")) - self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, "dir/file")) - self.failUnlessReallyEqual(ga1(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga1(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, "dir/file")) + self.failUnlessReallyEqual(ga1(u"bare"), (TA, b"bare")) + self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, b"baredir/file")) + self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, b"baredir/file:7")) + self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, b"dir/file")) + self.failUnlessReallyEqual(ga1(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga1(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, b"dir/file")) # default != None means we really expect a tahoe path, regardless of # whether we're on windows or not. This is what 'tahoe get' uses. - self.failUnlessReallyEqual(ga1(u"c:"), (CA, "")) - self.failUnlessReallyEqual(ga1(u"c:file"), (CA, "file")) - self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, "dir/file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga1(u"c:"), (CA, b"")) + self.failUnlessReallyEqual(ga1(u"c:file"), (CA, b"file")) + self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, b"dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file")) + self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir/file") def ga2(path): return get_alias(aliases, path, None) - self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, b"bare")) self.failUnlessReallyEqual(ga2(u"baredir/file"), - (DefaultAliasMarker, "baredir/file")) + (DefaultAliasMarker, b"baredir/file")) self.failUnlessReallyEqual(ga2(u"baredir/file:7"), - (DefaultAliasMarker, "baredir/file:7")) + (DefaultAliasMarker, b"baredir/file:7")) self.failUnlessReallyEqual(ga2(u"baredir/sub:1/file:7"), - (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, "dir/file")) + (DefaultAliasMarker, b"baredir/sub:1/file:7")) + self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, b"dir/file")) # on windows, we really want c:foo to indicate a local file. # default==None is what 'tahoe cp' uses. if uses_lettercolon: - self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, "c:")) - self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, "c:file")) + self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, b"c:")) + self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, b"c:file")) self.failUnlessReallyEqual(ga2(u"c:dir/file"), - (DefaultAliasMarker, "c:dir/file")) + (DefaultAliasMarker, b"c:dir/file")) else: - self.failUnlessReallyEqual(ga2(u"c:"), (CA, "")) - self.failUnlessReallyEqual(ga2(u"c:file"), (CA, "file")) - self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, "dir/file")) - self.failUnlessReallyEqual(ga2(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga2(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, "dir/file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file")) - self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + self.failUnlessReallyEqual(ga2(u"c:"), (CA, b"")) + self.failUnlessReallyEqual(ga2(u"c:file"), (CA, b"file")) + self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, b"dir/file")) + self.failUnlessReallyEqual(ga2(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga2(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, b"dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file")) + self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir/file") @@ -438,26 +454,26 @@ class CLI(CLITestMixin, unittest.TestCase): finally: common.pretend_platform_uses_lettercolon = old return retval - self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, "bare")) + self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, b"bare")) self.failUnlessReallyEqual(ga3(u"baredir/file"), - (DefaultAliasMarker, "baredir/file")) + (DefaultAliasMarker, b"baredir/file")) self.failUnlessReallyEqual(ga3(u"baredir/file:7"), - (DefaultAliasMarker, "baredir/file:7")) + (DefaultAliasMarker, b"baredir/file:7")) self.failUnlessReallyEqual(ga3(u"baredir/sub:1/file:7"), - (DefaultAliasMarker, "baredir/sub:1/file:7")) - self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, "")) - self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, "file")) - self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, "dir/file")) - self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, "c:")) - self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, "c:file")) + (DefaultAliasMarker, b"baredir/sub:1/file:7")) + self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, b"")) + self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, b"file")) + self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, b"dir/file")) + self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, b"c:")) + self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, b"c:file")) self.failUnlessReallyEqual(ga3(u"c:dir/file"), - (DefaultAliasMarker, "c:dir/file")) - self.failUnlessReallyEqual(ga3(u"work:"), (WA, "")) - self.failUnlessReallyEqual(ga3(u"work:file"), (WA, "file")) - self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, "dir/file")) - self.failUnlessReallyEqual(ga3(u"URI:stuff"), ("URI:stuff", "")) - self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), ("URI:stuff", "file")) - self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file")) + (DefaultAliasMarker, b"c:dir/file")) + self.failUnlessReallyEqual(ga3(u"work:"), (WA, b"")) + self.failUnlessReallyEqual(ga3(u"work:file"), (WA, b"file")) + self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, b"dir/file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff"), (b"URI:stuff", b"")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), (b"URI:stuff", b"file")) + self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file")) self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:") self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir") self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir/file") @@ -480,14 +496,14 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnlessRaises(common.UnknownAliasError, ga5, u"C:\\Windows") def test_alias_tolerance(self): - def s128(c): return base32.b2a(c*(128/8)) - def s256(c): return base32.b2a(c*(256/8)) - TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T")) + def s128(c): return base32.b2a(c*(128//8)) + def s256(c): return base32.b2a(c*(256//8)) + TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T")) aliases = {"present": TA, - "future": "URI-FROM-FUTURE:ooh:aah"} + "future": b"URI-FROM-FUTURE:ooh:aah"} def ga1(path): return get_alias(aliases, path, u"tahoe") - self.failUnlessReallyEqual(ga1(u"present:file"), (TA, "file")) + self.failUnlessReallyEqual(ga1(u"present:file"), (TA, b"file")) # this throws, via assert IDirnodeURI.providedBy(), since get_alias() # wants a dirnode, and the future cap gives us UnknownURI instead. self.failUnlessRaises(AssertionError, ga1, u"future:stuff") @@ -502,9 +518,9 @@ class CLI(CLITestMixin, unittest.TestCase): fileutil.make_dirs(basedir) for name in filenames: - open(os.path.join(unicode(basedir), name), "wb").close() + open(os.path.join(str(basedir), name), "wb").close() - for file in listdir_unicode(unicode(basedir)): + for file in listdir_unicode(str(basedir)): self.failUnlessIn(normalize(file), filenames) def test_exception_catcher(self): @@ -671,7 +687,7 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) # Make sure that validation extends to the "to" parameter d.addCallback(lambda ign: self.do_cli("create-alias", "havasu")) @@ -718,8 +734,9 @@ class Admin(unittest.TestCase): self.failUnlessEqual(pubkey_bits[0], vk_header, lines[1]) self.failUnless(privkey_bits[1].startswith("priv-v0-"), lines[0]) self.failUnless(pubkey_bits[1].startswith("pub-v0-"), lines[1]) - sk, pk = ed25519.signing_keypair_from_string(privkey_bits[1]) - vk_bytes = pubkey_bits[1] + sk, pk = ed25519.signing_keypair_from_string( + privkey_bits[1].encode("ascii")) + vk_bytes = pubkey_bits[1].encode("ascii") self.assertEqual( ed25519.string_from_verifying_key(pk), vk_bytes, @@ -729,8 +746,8 @@ class Admin(unittest.TestCase): def test_derive_pubkey(self): priv_key, pub_key = ed25519.create_signing_keypair() - priv_key_str = ed25519.string_from_signing_key(priv_key) - pub_key_str = ed25519.string_from_verifying_key(pub_key) + priv_key_str = str(ed25519.string_from_signing_key(priv_key), "ascii") + pub_key_str = str(ed25519.string_from_verifying_key(pub_key), "ascii") d = run_cli("admin", "derive-pubkey", priv_key_str) def _done(args): (rc, stdout, stderr) = args @@ -753,11 +770,11 @@ class Errors(GridTestMixin, CLITestMixin, unittest.TestCase): self.set_up_grid() c0 = self.g.clients[0] self.fileurls = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA, convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_bad(ur): self.uri_1share = ur.get_uri() - self.delete_shares_numbered(ur.get_uri(), range(1,10)) + self.delete_shares_numbered(ur.get_uri(), list(range(1,10))) d.addCallback(_stash_bad) # the download is abandoned as soon as it's clear that we won't get @@ -821,7 +838,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -836,7 +853,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -853,7 +870,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -868,7 +885,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -883,7 +900,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn("URI:", out) d.addCallback(_check) @@ -896,7 +913,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args, st): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn(st, out) return out @@ -932,7 +949,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args, st): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn(st, out) return out d.addCallback(_check, "URI:DIR2") @@ -976,7 +993,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessIn("URI:", out) d.addCallback(_check) @@ -992,7 +1009,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1016,7 +1033,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) d.addCallback(lambda ign: self.do_cli(self.command, "afile")) @@ -1034,7 +1051,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) d.addCallback(lambda ign: self.do_cli(self.command, "nonexistent:afile")) @@ -1060,7 +1077,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("'tahoe %s'" % (self.command,), err) self.failUnlessIn("path must be given", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1081,7 +1098,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 0", lines) @@ -1105,7 +1122,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1119,7 +1136,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1136,7 +1153,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(out, "") d.addCallback(_check) return d @@ -1144,7 +1161,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): # TODO: replace with @patch that supports Deferreds. import webbrowser def call_webbrowser_open(url): - self.failUnlessIn(self.alias_uri.replace(':', '%3A'), url) + self.failUnlessIn(str(self.alias_uri, "ascii").replace(':', '%3A'), url) self.webbrowser_open_called = True def _cleanup(res): webbrowser.open = self.old_webbrowser_open @@ -1161,15 +1178,15 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0, repr((rc, out, err))) self.failUnlessIn("Alias 'alias' created", out) - self.failUnlessReallyEqual(err, "") + self.assertEqual(err, "") self.alias_uri = get_aliases(self.get_clientdir())["alias"] d.addCallback(_check_alias) d.addCallback(lambda res: self.do_cli("webopen", "alias:")) def _check_webopen(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0, repr((rc, out, err))) - self.failUnlessReallyEqual(out, "") - self.failUnlessReallyEqual(err, "") + self.assertEqual(out, "") + self.assertEqual(err, "") self.failUnless(self.webbrowser_open_called) d.addCallback(_check_webopen) d.addBoth(_cleanup) @@ -1195,31 +1212,31 @@ class Options(ReallyEqualMixin, unittest.TestCase): fileutil.make_dirs("cli/test_options") fileutil.make_dirs("cli/test_options/private") fileutil.write("cli/test_options/node.url", "http://localhost:8080/\n") - filenode_uri = uri.WriteableSSKFileURI(writekey="\x00"*16, - fingerprint="\x00"*32) + filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x00"*16, + fingerprint=b"\x00"*32) private_uri = uri.DirectoryURI(filenode_uri).to_string() - fileutil.write("cli/test_options/private/root_dir.cap", private_uri + "\n") + fileutil.write("cli/test_options/private/root_dir.cap", private_uri + b"\n") def parse2(args): return parse_options("cli/test_options", "ls", args) o = parse2([]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") o = parse2(["--node-url", "http://example.org:8111/"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") # -u for --node-url used to clash with -u for --uri (tickets #1949 and #2137). o = parse2(["-u", "http://example.org:8111/"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") self.failIf(o["uri"]) o = parse2(["-u", "http://example.org:8111/", "--uri"]) self.failUnlessEqual(o['node-url'], "http://example.org:8111/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri) self.failUnlessEqual(o.where, u"") self.failUnless(o["uri"]) @@ -1228,17 +1245,17 @@ class Options(ReallyEqualMixin, unittest.TestCase): self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], "root") self.failUnlessEqual(o.where, u"") - other_filenode_uri = uri.WriteableSSKFileURI(writekey="\x11"*16, - fingerprint="\x11"*32) + other_filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x11"*16, + fingerprint=b"\x11"*32) other_uri = uri.DirectoryURI(other_filenode_uri).to_string() o = parse2(["--dir-cap", other_uri]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri) self.failUnlessEqual(o.where, u"") o = parse2(["--dir-cap", other_uri, "subdir"]) self.failUnlessEqual(o['node-url'], "http://localhost:8080/") - self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri) + self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri) self.failUnlessEqual(o.where, u"subdir") self.failUnlessRaises(usage.UsageError, parse2, @@ -1325,7 +1342,7 @@ class Run(unittest.TestCase): If the pidfile exists but does not contain a numeric value, a complaint to this effect is written to stderr. """ - basedir = FilePath(self.mktemp().decode("ascii")) + basedir = FilePath(ensure_str(self.mktemp())) basedir.makedirs() basedir.child(u"twistd.pid").setContent(b"foo") basedir.child(u"tahoe-client.tac").setContent(b"") @@ -1333,7 +1350,7 @@ class Run(unittest.TestCase): config = tahoe_run.RunOptions() config.stdout = StringIO() config.stderr = StringIO() - config['basedir'] = basedir.path + config['basedir'] = ensure_text(basedir.path) config.twistd_args = [] result_code = tahoe_run.run(config) diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py index 6cebec4a5..fff50f331 100644 --- a/src/allmydata/test/cli/test_cp.py +++ b/src/allmydata/test/cli/test_cp.py @@ -1,4 +1,14 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path, json from twisted.trial import unittest @@ -24,12 +34,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def test_unicode_filename(self): self.basedir = "cli/Cp/unicode_filename" - fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall") - try: - fn1_arg = fn1.encode(get_io_encoding()) - artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + fn1 = os.path.join(self.basedir, u"\u00C4rtonwall") + artonwall_arg = u"\u00C4rtonwall" skip_if_cannot_represent_filename(fn1) @@ -44,15 +50,20 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): d = self.do_cli("create-alias", "tahoe") - d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:")) + d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg)) - d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1)) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1)) + + # Version where destination filename is explicitly Unicode too. + d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:" + artonwall_arg + "-2")) + d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg + "-2")) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1)) d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:")) d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica")) - d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) + d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA2)) d.addCallback(lambda res: self.do_cli("ls", "tahoe:")) def _check(args): @@ -66,8 +77,10 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n") - self.failUnlessReallyEqual(err, "") + if PY2: + out = out.decode(get_io_encoding()) + self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n\u00C4rtonwall-2\n") + self.assertEqual(len(err), 0, err) d.addCallback(_check) return d @@ -98,7 +111,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): fn1 = os.path.join(self.basedir, "Metallica") fn2 = os.path.join(outdir, "Not Metallica") fn3 = os.path.join(outdir, "test2") - DATA1 = "puppies" * 10000 + DATA1 = b"puppies" * 10000 fileutil.write(fn1, DATA1) d = self.do_cli("create-alias", "tahoe") @@ -128,7 +141,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("when copying into a directory, all source files must have names, but", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_resp) # Create a directory, linked at tahoe:test . @@ -200,13 +213,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): def test_unicode_dirnames(self): self.basedir = "cli/Cp/unicode_dirnames" - fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall") - try: - fn1_arg = fn1.encode(get_io_encoding()) - del fn1_arg # hush pyflakes - artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + fn1 = os.path.join(self.basedir, u"\u00C4rtonwall") + artonwall_arg = u"\u00C4rtonwall" skip_if_cannot_represent_filename(fn1) @@ -222,17 +230,79 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase): unicode_to_output(u"\u00C4rtonwall") except UnicodeEncodeError: self.failUnlessReallyEqual(rc, 1) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err) self.failUnlessIn("files whose names could not be converted", err) else: self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n") - self.failUnlessReallyEqual(err, "") + if PY2: + out = out.decode(get_io_encoding()) + self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n") + self.assertEqual(len(err), 0, err) d.addCallback(_check) return d + @defer.inlineCallbacks + def test_cp_duplicate_directories(self): + self.basedir = "cli/Cp/cp_duplicate_directories" + self.set_up_grid(oneshare=True) + + filename = os.path.join(self.basedir, "file") + data = b"abc\xff\x00\xee" + with open(filename, "wb") as f: + f.write(data) + + yield self.do_cli("create-alias", "tahoe") + (rc, out, err) = yield self.do_cli("mkdir", "tahoe:test1") + self.assertEqual(rc, 0, (rc, err)) + dircap = out.strip() + + (rc, out, err) = yield self.do_cli("cp", filename, "tahoe:test1/file") + self.assertEqual(rc, 0, (rc, err)) + + # Now duplicate dirnode, testing duplicates on destination side: + (rc, out, err) = yield self.do_cli( + "cp", "--recursive", dircap, "tahoe:test2/") + self.assertEqual(rc, 0, (rc, err)) + (rc, out, err) = yield self.do_cli( + "cp", "--recursive", dircap, "tahoe:test3/") + self.assertEqual(rc, 0, (rc, err)) + + # Now copy to local directory, testing duplicates on origin side: + yield self.do_cli("cp", "--recursive", "tahoe:", self.basedir) + + for i in range(1, 4): + with open(os.path.join(self.basedir, "test%d" % (i,), "file"), "rb") as f: + self.assertEquals(f.read(), data) + + @defer.inlineCallbacks + def test_cp_immutable_file(self): + self.basedir = "cli/Cp/cp_immutable_file" + self.set_up_grid(oneshare=True) + + filename = os.path.join(self.basedir, "source_file") + data = b"abc\xff\x00\xee" + with open(filename, "wb") as f: + f.write(data) + + # Create immutable file: + yield self.do_cli("create-alias", "tahoe") + (rc, out, _) = yield self.do_cli("put", filename, "tahoe:file1") + filecap = out.strip() + self.assertEqual(rc, 0) + + # Copy it: + (rc, _, _) = yield self.do_cli("cp", "tahoe:file1", "tahoe:file2") + self.assertEqual(rc, 0) + + # Make sure resulting file is the same: + (rc, _, _) = yield self.do_cli("cp", "--recursive", "--caps-only", + "tahoe:", self.basedir) + self.assertEqual(rc, 0) + with open(os.path.join(self.basedir, "file2")) as f: + self.assertEqual(f.read().strip(), filecap) + def test_cp_replaces_mutable_file_contents(self): self.basedir = "cli/Cp/cp_replaces_mutable_file_contents" self.set_up_grid(oneshare=True) @@ -818,9 +888,9 @@ cp -r $DIRCAP5 $DIRCAP6 to : E9-COLLIDING-TARGETS """ class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase): - FILE_CONTENTS = "file text" - FILE_CONTENTS_5 = "5" - FILE_CONTENTS_6 = "6" + FILE_CONTENTS = b"file text" + FILE_CONTENTS_5 = b"5" + FILE_CONTENTS_6 = b"6" def do_setup(self): # first we build a tahoe filesystem that contains: diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index ea3200e2e..176bf7576 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -1,12 +1,25 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six.moves import StringIO import os.path from twisted.trial import unittest -import urllib +from urllib.parse import quote as url_quote + from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output, get_io_encoding +from allmydata.util.encodingutil import quote_output_u from .common import CLITestMixin class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -22,7 +35,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): rc = cli.webopen(o.subOptions, urls.append) self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(len(urls), 1) - self.failUnlessReallyEqual(urls[0], expected_url) + self.assertEqual(urls[0], expected_url) def test_create(self): self.basedir = "cli/CreateAlias/create" @@ -36,19 +49,19 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.assertIn("Alias 'tahoe' created", stdout) aliases = get_aliases(self.get_clientdir()) self.failUnless("tahoe" in aliases) - self.failUnless(aliases["tahoe"].startswith("URI:DIR2:")) + self.failUnless(aliases["tahoe"].startswith(b"URI:DIR2:")) d.addCallback(_done) d.addCallback(lambda res: self.do_cli("create-alias", "two:")) def _stash_urls(res): aliases = get_aliases(self.get_clientdir()) node_url_file = os.path.join(self.get_clientdir(), "node.url") - nodeurl = fileutil.read(node_url_file).strip() + nodeurl = fileutil.read(node_url_file, mode="r").strip() self.welcome_url = nodeurl uribase = nodeurl + "uri/" - self.tahoe_url = uribase + urllib.quote(aliases["tahoe"]) + self.tahoe_url = uribase + url_quote(aliases["tahoe"]) self.tahoe_subdir_url = self.tahoe_url + "/subdir" - self.two_url = uribase + urllib.quote(aliases["two"]) + self.two_url = uribase + url_quote(aliases["two"]) self.two_uri = aliases["two"] d.addCallback(_stash_urls) @@ -128,13 +141,13 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): # like a valid dircap, so get_aliases() will raise an exception. aliases = get_aliases(self.get_clientdir()) self.failUnless("added" in aliases) - self.failUnless(aliases["added"].startswith("URI:DIR2:")) + self.failUnless(aliases["added"].startswith(b"URI:DIR2:")) # to be safe, let's confirm that we don't see "NAME2:" in CAP1. # No chance of a false-negative, because the hyphen in # "un-corrupted1" is not a valid base32 character. - self.failIfIn("un-corrupted1:", aliases["added"]) + self.failIfIn(b"un-corrupted1:", aliases["added"]) self.failUnless("un-corrupted1" in aliases) - self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:")) + self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:")) d.addCallback(_check_not_corrupted1) def _remove_trailing_newline_and_add_alias(ign): @@ -149,10 +162,10 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIf(stderr) aliases = get_aliases(self.get_clientdir()) self.failUnless("un-corrupted1" in aliases) - self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:")) - self.failIfIn("un-corrupted2:", aliases["un-corrupted1"]) + self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:")) + self.failIfIn(b"un-corrupted2:", aliases["un-corrupted1"]) self.failUnless("un-corrupted2" in aliases) - self.failUnless(aliases["un-corrupted2"].startswith("URI:DIR2:")) + self.failUnless(aliases["un-corrupted2"].startswith(b"URI:DIR2:")) d.addCallback(_check_not_corrupted) return d @@ -160,61 +173,62 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/CreateAlias/create_unicode" self.set_up_grid(oneshare=True) - try: - etudes_arg = u"\u00E9tudes".encode(get_io_encoding()) - lumiere_arg = u"lumi\u00E8re.txt".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + etudes_arg = u"\u00E9tudes" + lumiere_arg = u"lumi\u00E8re.txt" d = self.do_cli("create-alias", etudes_arg) def _check_create_unicode(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out) + self.assertEqual(len(err), 0, err) + self.failUnlessIn(u"Alias %s created" % (quote_output_u(etudes_arg),), out) aliases = get_aliases(self.get_clientdir()) - self.failUnless(aliases[u"\u00E9tudes"].startswith("URI:DIR2:")) + self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) d.addCallback(_check_create_unicode) d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) def _check_ls1(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(err), 0, err) + self.assertEqual(len(out), 0, out) d.addCallback(_check_ls1) + DATA = b"Blah blah blah \xff blah \x00 blah" d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":uploaded.txt", - stdin="Blah blah blah")) + stdin=DATA)) d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":")) def _check_ls2(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "uploaded.txt\n") + self.assertEqual(len(err), 0, err) + self.assertEqual(out, "uploaded.txt\n") d.addCallback(_check_ls2) - d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt")) + d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt", + return_bytes=True)) def _check_get(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "Blah blah blah") + self.assertEqual(len(err), 0, err) + self.failUnlessReallyEqual(out, DATA) d.addCallback(_check_get) # Ensure that an Unicode filename in an Unicode alias works as expected d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":" + lumiere_arg, - stdin="Let the sunshine In!")) + stdin=b"Let the sunshine In!")) - d.addCallback(lambda res: self.do_cli("get", - get_aliases(self.get_clientdir())[u"\u00E9tudes"] + "/" + lumiere_arg)) + d.addCallback(lambda res: self.do_cli( + "get", + str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg, + return_bytes=True)) def _check_get2(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(out, "Let the sunshine In!") + self.assertEqual(len(err), 0, err) + self.failUnlessReallyEqual(out, b"Let the sunshine In!") d.addCallback(_check_get2) return d diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index f356e18de..20d012995 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,8 +1,25 @@ +""" +Ported to Pythn 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os import mock import json from os.path import join +try: + from typing import Optional, Sequence +except ImportError: + pass + from twisted.trial import unittest from twisted.internet import defer from ..common_util import run_cli @@ -16,6 +33,8 @@ class _FakeWormhole(object): def __init__(self, outgoing_messages): self.messages = [] + for o in outgoing_messages: + assert isinstance(o, bytes) self._outgoing = outgoing_messages def get_code(self): @@ -26,15 +45,16 @@ class _FakeWormhole(object): def get_welcome(self): return defer.succeed( - json.dumps({ + { u"welcome": {}, - }) + } ) def allocate_code(self): return None def send_message(self, msg): + assert isinstance(msg, bytes) self.messages.append(msg) def get_message(self): @@ -45,6 +65,10 @@ class _FakeWormhole(object): def _create_fake_wormhole(outgoing_messages): + outgoing_messages = [ + m.encode("utf-8") if isinstance(m, str) else m + for m in outgoing_messages + ] return _FakeWormhole(outgoing_messages) @@ -144,17 +168,27 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - @defer.inlineCallbacks - def test_invite_success(self): + def _invite_success(self, extra_args=(), tahoe_config=None): + # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred """ - successfully send an invite + Exercise an expected-success case of ``tahoe invite``. + + :param extra_args: Positional arguments to pass to ``tahoe invite`` + before the nickname. + + :param tahoe_config: If given, bytes to write to the node's + ``tahoe.cfg`` before running ``tahoe invite. """ intro_dir = os.path.join(self.basedir, "introducer") # we've never run the introducer, so it hasn't created # introducer.furl yet priv_dir = join(intro_dir, "private") - with open(join(priv_dir, "introducer.furl"), "w") as f: - f.write("pb://fooblam\n") + with open(join(priv_dir, "introducer.furl"), "w") as fobj_intro: + fobj_intro.write("pb://fooblam\n") + if tahoe_config is not None: + assert isinstance(tahoe_config, bytes) + with open(join(intro_dir, "tahoe.cfg"), "wb") as fobj_cfg: + fobj_cfg.write(tahoe_config) with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: fake_wh = _create_fake_wormhole([ @@ -162,34 +196,79 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ]) w.create = mock.Mock(return_value=fake_wh) - rc, out, err = yield run_cli( + extra_args = tuple(extra_args) + + d = run_cli( "-d", intro_dir, "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - "foo", + *(extra_args + ("foo",)) ) - self.assertEqual(2, len(fake_wh.messages)) - self.assertEqual( - json.loads(fake_wh.messages[0]), - { - "abilities": + + def done(result): + rc, out, err = result + self.assertEqual(2, len(fake_wh.messages)) + self.assertEqual( + json.loads(fake_wh.messages[0]), { - "server-v1": {} + "abilities": + { + "server-v1": {} + }, }, - }, - ) - self.assertEqual( - json.loads(fake_wh.messages[1]), - { - "shares-needed": "1", - "shares-total": "1", - "nickname": "foo", - "introducer": "pb://fooblam", - "shares-happy": "1", - }, - ) + ) + invite = json.loads(fake_wh.messages[1]) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) + return invite + d.addCallback(done) + return d + + @defer.inlineCallbacks + def test_invite_success(self): + """ + successfully send an invite + """ + invite = yield self._invite_success(( + "--shares-needed", "1", + "--shares-happy", "2", + "--shares-total", "3", + )) + self.assertEqual( + invite["shares-needed"], "1", + ) + self.assertEqual( + invite["shares-happy"], "2", + ) + self.assertEqual( + invite["shares-total"], "3", + ) + + @defer.inlineCallbacks + def test_invite_success_read_share_config(self): + """ + If ``--shares-{needed,happy,total}`` are not given on the command line + then the invitation is generated using the configured values. + """ + invite = yield self._invite_success(tahoe_config=b""" +[client] +shares.needed = 2 +shares.happy = 4 +shares.total = 6 +""") + self.assertEqual( + invite["shares-needed"], "2", + ) + self.assertEqual( + invite["shares-happy"], "4", + ) + self.assertEqual( + invite["shares-total"], "6", + ) + @defer.inlineCallbacks def test_invite_no_furl(self): diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index fff57cdc9..1206579f1 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_str + from twisted.trial import unittest from twisted.internet import defer @@ -8,61 +21,52 @@ from ..no_network import GridTestMixin from allmydata.util.encodingutil import quote_output, get_io_encoding from .common import CLITestMixin + class List(GridTestMixin, CLITestMixin, unittest.TestCase): def test_list(self): self.basedir = "cli/List/list" self.set_up_grid() c0 = self.g.clients[0] - small = "small" + small = b"small" - # u"g\u00F6\u00F6d" might not be representable in the argv and/or output encodings. - # It is initially included in the directory in any case. - try: - good_arg = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_arg = None + good_arg = u"g\u00F6\u00F6d" + good_out = u"g\u00F6\u00F6d" - try: - good_out = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_out = None + # On Python 2 we get bytes, so we need encoded version. On Python 3 + # stdio is unicode so can leave unchanged. + good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding()) d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence="")) + self.rooturi = str(n.get_uri(), "utf-8") + return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_goodcap(n): self.goodcap = n.get_uri() d.addCallback(_stash_goodcap) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"1share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(1,10))) + self.delete_shares_numbered(n.get_uri(), list(range(1,10)))) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"0share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(0,10))) + self.delete_shares_numbered(n.get_uri(), list(range(0,10)))) d.addCallback(lambda ign: self.do_cli("add-alias", "tahoe", self.rooturi)) d.addCallback(lambda ign: self.do_cli("ls")) def _check1(args): (rc, out, err) = args - if good_out is None: - self.failUnlessReallyEqual(rc, 1) - self.failUnlessIn("files whose names could not be converted", err) - self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share"])) - else: - self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", good_out])) + self.failUnlessReallyEqual(rc, 0) + self.assertEqual(len(err), 0, err) + expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded]) + self.assertEqual(sorted(out.splitlines()), expected) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) def _check2(args): (rc, out, err) = args self.failIfEqual(rc, 0) - self.failUnlessReallyEqual(err.strip(), "No such file or directory") - self.failUnlessReallyEqual(out, "") + self.assertEqual(err.strip(), "No such file or directory") + self.assertEqual(len(out), 0, out) d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("ls", "1share")) def _check3(args): @@ -72,7 +76,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("UnrecoverableFileError:", err) self.failUnlessIn("could not be retrieved, because there were " "insufficient good shares.", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check3) d.addCallback(lambda ign: self.do_cli("ls", "0share")) d.addCallback(_check3) @@ -82,13 +86,13 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("files whose names could not be converted", err) self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) else: # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) - self.failUnlessIn(good_out, out) - self.failIfIn("-r-- %d -" % len(small), out, + self.failUnlessIn(good_out_encoded, out) + self.failIfIn(ensure_str("-r-- %d -" % len(small)), out, "trailing hyphen means unknown date") if good_arg is not None: @@ -106,7 +110,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # metadata, just the size (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip()) + self.assertEqual("-r-- %d -" % len(small), out.strip()) d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap)) d.addCallback(_check5) @@ -118,7 +122,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _check1_ascii(args): (rc,out,err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", "good"])) d.addCallback(_check1_ascii) def _check4_ascii(args): @@ -139,7 +143,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good")) d.addCallback(_check4_ascii) - unknown_immcap = "imm.URI:unknown" + unknown_immcap = b"imm.URI:unknown" def _create_unknown(ign): nm = c0.nodemaker kids = {u"unknownchild-imm": (nm.create_from_cap(unknown_immcap), {})} @@ -178,7 +182,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -193,7 +197,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -226,8 +230,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # The uploaders may run at the same time, so we need two # MutableData instances or they'll fight over offsets &c and # break. - mutable_data = MutableData("data" * 100000) - mutable_data2 = MutableData("data" * 100000) + mutable_data = MutableData(b"data" * 100000) + mutable_data2 = MutableData(b"data" * 100000) # Add both kinds of mutable node. d1 = nm.create_mutable_file(mutable_data, version=MDMF_VERSION) @@ -235,8 +239,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): version=SDMF_VERSION) # Add an immutable node. We do this through the directory, # with add_file. - immutable_data = upload.Data("immutable data" * 100000, - convergence="") + immutable_data = upload.Data(b"immutable data" * 100000, + convergence=b"") d3 = n.add_file(u"immutable", immutable_data) ds = [d1, d2, d3] dl = defer.DeferredList(ds) @@ -294,12 +298,12 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _got_json(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) - self.failUnlessEqual(err, "") - self.failUnlessIn(self._mdmf_uri, out) - self.failUnlessIn(self._mdmf_readonly_uri, out) - self.failUnlessIn(self._sdmf_uri, out) - self.failUnlessIn(self._sdmf_readonly_uri, out) - self.failUnlessIn(self._imm_uri, out) + self.assertEqual(len(err), 0, err) + self.failUnlessIn(str(self._mdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._mdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._imm_uri, "ascii"), out) self.failUnlessIn('"format": "SDMF"', out) self.failUnlessIn('"format": "MDMF"', out) d.addCallback(_got_json) diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index 9d1a64974..0bb9ba369 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from allmydata.util import fileutil @@ -5,15 +17,16 @@ from ..no_network import GridTestMixin from allmydata.scripts import tahoe_mv from .common import CLITestMixin + class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): def test_mv_behavior(self): self.basedir = "cli/Mv/mv_behavior" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) fn2 = os.path.join(self.basedir, "file2") - DATA2 = "UML diagrams" + DATA2 = b"UML diagrams" fileutil.write(fn2, DATA2) # copy both files to the grid d = self.do_cli("create-alias", "tahoe") @@ -104,11 +117,11 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Mv/mv_error_if_DELETE_fails" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) original_do_http = tahoe_mv.do_http - def mock_do_http(method, url, body=""): + def mock_do_http(method, url, body=b""): if method == "DELETE": class FakeResponse(object): def read(self): @@ -152,7 +165,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first @@ -180,7 +193,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("fake", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py index 3392e67b4..03306ab71 100644 --- a/src/allmydata/test/cli/test_put.py +++ b/src/allmydata/test/cli/test_put.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from twisted.python import usage @@ -17,7 +29,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): # tahoe get `echo DATA | tahoe put` # tahoe get `echo DATA | tahoe put -` self.basedir = "cli/Put/unlinked_immutable_stdin" - DATA = "data" * 100 + DATA = b"data\xff" * 100 self.set_up_grid(oneshare=True) d = self.do_cli("put", stdin=DATA) def _uploaded(res): @@ -27,10 +39,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.readcap = out self.failUnless(self.readcap.startswith("URI:CHK:")) d.addCallback(_uploaded) - d.addCallback(lambda res: self.do_cli("get", self.readcap)) + d.addCallback(lambda res: self.do_cli("get", self.readcap, + return_bytes=True)) def _downloaded(res): (rc, out, err) = res - self.failUnlessReallyEqual(err, "") + self.failUnlessReallyEqual(err, b"") self.failUnlessReallyEqual(out, DATA) d.addCallback(_downloaded) d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA)) @@ -46,10 +59,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/unlinked_immutable_from_file" self.set_up_grid(oneshare=True) - rel_fn = unicode(os.path.join(self.basedir, "DATAFILE")) + rel_fn = str(os.path.join(self.basedir, "DATAFILE")) abs_fn = abspath_expanduser_unicode(rel_fn) # we make the file small enough to fit in a LIT file, for speed - fileutil.write(rel_fn, "short file") + fileutil.write(rel_fn, b"short file has some bytes \xff yes") d = self.do_cli_unicode(u"put", [rel_fn]) def _uploaded(args): (rc, out, err) = args @@ -79,8 +92,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): rel_fn = os.path.join(self.basedir, "DATAFILE") # we make the file small enough to fit in a LIT file, for speed - DATA = "short file" - DATA2 = "short file two" + DATA = b"short file" + DATA2 = b"short file two" fileutil.write(rel_fn, DATA) d = self.do_cli("create-alias", "tahoe") @@ -95,7 +108,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.readcap = readcap d.addCallback(_uploaded) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded.txt")) + self.do_cli("get", "tahoe:uploaded.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -110,32 +124,36 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda res: self.do_cli("put", rel_fn, "subdir/uploaded2.txt")) - d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt")) + d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:uploaded3.txt")) - d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt")) + d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) d.addCallback(lambda res: self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:subdir/uploaded4.txt")) + self.do_cli("get", "tahoe:subdir/uploaded4.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) def _get_dircap(res): - self.dircap = get_aliases(self.get_clientdir())["tahoe"] + self.dircap = str(get_aliases(self.get_clientdir())["tahoe"], "ascii") d.addCallback(_get_dircap) d.addCallback(lambda res: self.do_cli("put", rel_fn, self.dircap+":./uploaded5.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded5.txt")) + self.do_cli("get", "tahoe:uploaded5.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -143,7 +161,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("put", rel_fn, self.dircap+":./subdir/uploaded6.txt")) d.addCallback(lambda res: - self.do_cli("get", "tahoe:subdir/uploaded6.txt")) + self.do_cli("get", "tahoe:subdir/uploaded6.txt", + return_bytes=True)) d.addCallback(lambda rc_stdout_stderr: self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA)) @@ -158,10 +177,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/mutable_unlinked" self.set_up_grid(oneshare=True) - DATA = "data" * 100 - DATA2 = "two" * 100 + DATA = b"data" * 100 + DATA2 = b"two" * 100 rel_fn = os.path.join(self.basedir, "DATAFILE") - DATA3 = "three" * 100 + DATA3 = b"three" * 100 fileutil.write(rel_fn, DATA3) d = self.do_cli("put", "--mutable", stdin=DATA) @@ -172,7 +191,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.filecap = out self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap) d.addCallback(_created) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA)) d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2)) @@ -182,7 +201,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("200 OK", err) self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap)) @@ -191,7 +210,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("200 OK", err) self.failUnlessReallyEqual(self.filecap, out) d.addCallback(_replaced2) - d.addCallback(lambda res: self.do_cli("get", self.filecap)) + d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA3)) return d @@ -204,10 +223,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Put/mutable" self.set_up_grid(oneshare=True) - DATA1 = "data" * 100 + DATA1 = b"data" * 100 fn1 = os.path.join(self.basedir, "DATA1") fileutil.write(fn1, DATA1) - DATA2 = "two" * 100 + DATA2 = b"two\xff" * 100 fn2 = os.path.join(self.basedir, "DATA2") fileutil.write(fn2, DATA2) @@ -229,7 +248,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessEqual(out, self.uri, str(res)) d.addCallback(_check2) d.addCallback(lambda res: - self.do_cli("get", "tahoe:uploaded.txt")) + self.do_cli("get", "tahoe:uploaded.txt", return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2)) return d @@ -429,26 +448,23 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d def test_immutable_from_file_unicode(self): # tahoe put "\u00E0 trier.txt" "\u00E0 trier.txt" - try: - a_trier_arg = u"\u00E0 trier.txt".encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") + a_trier_arg = u"\u00E0 trier.txt" skip_if_cannot_represent_filename(u"\u00E0 trier.txt") self.basedir = "cli/Put/immutable_from_file_unicode" self.set_up_grid(oneshare=True) - rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt") + rel_fn = os.path.join(str(self.basedir), u"\u00E0 trier.txt") # we make the file small enough to fit in a LIT file, for speed - DATA = "short file" + DATA = b"short file \xff bytes" fileutil.write(rel_fn, DATA) d = self.do_cli("create-alias", "tahoe") @@ -464,8 +480,26 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_uploaded) d.addCallback(lambda res: - self.do_cli("get", "tahoe:" + a_trier_arg)) + self.do_cli("get", "tahoe:" + a_trier_arg, + return_bytes=True)) d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA)) return d + + def test_no_leading_slash(self): + self.basedir = "cli/Put/leading_slash" + self.set_up_grid(oneshare=True) + + fn1 = os.path.join(self.basedir, "DATA1") + + d = self.do_cli("create-alias", "tahoe") + d.addCallback(lambda res: + self.do_cli("put", fn1, "tahoe:/uploaded.txt")) + def _check(args): + (rc, out, err) = args + self.assertEqual(rc, 1) + self.failUnlessIn("must not start with a slash", err) + self.assertEqual(len(out), 0, out) + d.addCallback(_check) + return d diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index d27791f34..6100d2568 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.scripts.tahoe_run``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import ( StringIO, @@ -50,7 +60,7 @@ class DaemonizeTheRealServiceTests(SyncTestCase): """ nodedir = FilePath(self.mktemp()) nodedir.makedirs() - nodedir.child("tahoe.cfg").setContent(config) + nodedir.child("tahoe.cfg").setContent(config.encode("ascii")) nodedir.child("tahoe-client.tac").touch() options = parse_options(["run", nodedir.path]) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index a04939429..724661211 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -37,37 +37,53 @@ from allmydata.util import jsonbytes as json from ..no_network import GridTestMixin from ..common_web import do_http -from ..status import FakeStatus from .common import CLITestMixin +class FakeStatus(object): + def __init__(self): + self.status = [] + + def setServiceParent(self, p): + pass + + def get_status(self): + return self.status + + def get_storage_index(self): + return None + + def get_size(self): + return None + + class ProgressBar(unittest.TestCase): def test_ascii0(self): - prog = pretty_progress(80.0, size=10, ascii=True) + prog = pretty_progress(80.0, size=10, output_ascii=True) self.assertEqual('########. ', prog) def test_ascii1(self): - prog = pretty_progress(10.0, size=10, ascii=True) + prog = pretty_progress(10.0, size=10, output_ascii=True) self.assertEqual('#. ', prog) def test_ascii2(self): - prog = pretty_progress(13.0, size=10, ascii=True) + prog = pretty_progress(13.0, size=10, output_ascii=True) self.assertEqual('#o ', prog) def test_ascii3(self): - prog = pretty_progress(90.0, size=10, ascii=True) + prog = pretty_progress(90.0, size=10, output_ascii=True) self.assertEqual('#########.', prog) def test_unicode0(self): self.assertEqual( - pretty_progress(82.0, size=10, ascii=False), + pretty_progress(82.0, size=10, output_ascii=False), u'\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u258e ', ) def test_unicode1(self): self.assertEqual( - pretty_progress(100.0, size=10, ascii=False), + pretty_progress(100.0, size=10, output_ascii=False), u'\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588', ) @@ -114,9 +130,10 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_check) return d - @mock.patch('sys.stdout') - def test_help(self, fake): - return self.do_cli('status', '--help') + @defer.inlineCallbacks + def test_help(self): + rc, _, _ = yield self.do_cli('status', '--help') + self.assertEqual(rc, 0) class CommandStatus(unittest.TestCase): diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 34d73a199..be0381e11 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "CLINodeAPI", @@ -81,7 +92,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): self._fds = fds def connectionMade(self): - for proto in self._fds.values(): + for proto in list(self._fds.values()): proto.makeConnection(self.transport) def childDataReceived(self, childFD, data): @@ -94,7 +105,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): def processEnded(self, reason): notified = set() - for proto in self._fds.values(): + for proto in list(self._fds.values()): if proto not in notified: proto.connectionLost(reason) notified.add(proto) @@ -143,6 +154,7 @@ class CLINodeAPI(object): exe = sys.executable argv = [ exe, + "-b", u"-m", u"allmydata.scripts.runner", ] + argv diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 94a59f5c9..0f2dc7c62 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,4 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import chr as byteschr __all__ = [ "SyncTestCase", @@ -15,8 +26,6 @@ __all__ = [ "PIPE", ] -from past.builtins import chr as byteschr, unicode - import sys import os, random, struct import six @@ -106,7 +115,7 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32": +if sys.platform == "win32" and PY2: # Python 2.7 doesn't have good options for launching a process with # non-ASCII in its command line. So use this alternative that does a # better job. However, only use it on Windows because it doesn't work @@ -253,7 +262,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) @@ -264,7 +273,7 @@ class UseNode(object): return "\n".join( " = ".join((key, value)) for (key, value) - in config.items() + in list(config.items()) ) if self.plugin_config is None: @@ -396,7 +405,7 @@ class SameProcessStreamEndpointAssigner(object): :return: A two-tuple of (location hint, port endpoint description) as strings. """ - if IReactorSocket.providedBy(reactor): + if sys.platform != "win32" and IReactorSocket.providedBy(reactor): # On this platform, we can reliable pre-allocate a listening port. # Once it is bound we know it will not fail later with EADDRINUSE. s = socket(AF_INET, SOCK_STREAM) @@ -539,8 +548,8 @@ class FakeCHKFileNode(object): # type: ignore # incomplete implementation return defer.succeed(self) - def download_to_data(self, progress=None): - return download_to_data(self, progress=progress) + def download_to_data(self): + return download_to_data(self) download_best_version = download_to_data @@ -717,11 +726,11 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation d.addCallback(_done) return d - def download_best_version(self, progress=None): - return defer.succeed(self._download_best_version(progress=progress)) + def download_best_version(self): + return defer.succeed(self._download_best_version()) - def _download_best_version(self, ignored=None, progress=None): + def _download_best_version(self, ignored=None): if isinstance(self.my_uri, uri.LiteralFileURI): return self.my_uri.data if self.storage_index not in self.all_contents: @@ -849,17 +858,17 @@ class WebErrorMixin(object): callable=None, *args, **kwargs): # returns a Deferred with the response body if isinstance(substring, bytes): - substring = unicode(substring, "ascii") - if isinstance(response_substring, unicode): + substring = str(substring, "ascii") + if isinstance(response_substring, str): response_substring = response_substring.encode("ascii") - assert substring is None or isinstance(substring, unicode) + assert substring is None or isinstance(substring, str) assert response_substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: - code_string = unicode(f) + code_string = str(f) self.failUnless(substring in code_string, "%s: substring '%s' not in '%s'" % (which, substring, code_string)) @@ -882,7 +891,7 @@ class WebErrorMixin(object): body = yield response.content() self.assertEquals(response.code, code) if response_substring is not None: - if isinstance(response_substring, unicode): + if isinstance(response_substring, str): response_substring = response_substring.encode("utf-8") self.assertIn(response_substring, body) returnValue(body) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 4b787d7a8..ec449c2af 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,14 +1,23 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals -from future.utils import PY2, bchr, binary_type +from future.utils import PY2, PY3, bchr, binary_type from future.builtins import str as future_str -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os +import sys import time import signal from random import randrange -from six.moves import StringIO +if PY2: + from StringIO import StringIO from io import ( TextIOWrapper, BytesIO, @@ -20,11 +29,11 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv def skip_if_cannot_represent_filename(u): - precondition(isinstance(u, unicode)) + precondition(isinstance(u, str)) enc = get_filesystem_encoding() if not unicode_platform(): @@ -33,13 +42,6 @@ def skip_if_cannot_represent_filename(u): except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") -def skip_if_cannot_represent_argv(u): - precondition(isinstance(u, unicode)) - try: - u.encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.") - def _getvalue(io): """ @@ -51,7 +53,7 @@ def _getvalue(io): def maybe_unicode_to_argv(o): """Convert object to argv form if necessary.""" - if isinstance(o, unicode): + if isinstance(o, str): return unicode_to_argv(o) return o @@ -64,23 +66,28 @@ def run_cli_native(verb, *args, **kwargs): Most code should prefer ``run_cli_unicode`` which deals with all the necessary encoding considerations. - :param native_str verb: The command to run. For example, ``"create-node"``. + :param native_str verb: The command to run. For example, + ``"create-node"``. - :param [native_str] args: The arguments to pass to the command. For example, - ``("--hostname=localhost",)``. + :param [native_str] args: The arguments to pass to the command. For + example, ``("--hostname=localhost",)``. - :param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable - before ``verb``. + :param [native_str] nodeargs: Extra arguments to pass to the Tahoe + executable before ``verb``. - :param native_str stdin: Text to pass to the command via stdin. + :param bytes|unicode stdin: Text or bytes to pass to the command via stdin. :param NoneType|str encoding: The name of an encoding which stdout and - stderr will be configured to use. ``None`` means stdout and stderr - will accept bytes and unicode and use the default system encoding for - translating between them. + stderr will be configured to use. ``None`` means matching default + behavior for the given Python version. + + :param bool return_bytes: If False, stdout/stderr is native string, + matching native behavior. If True, stdout/stderr are returned as + bytes. """ nodeargs = kwargs.pop("nodeargs", []) - encoding = kwargs.pop("encoding", None) + encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" + return_bytes = kwargs.pop("return_bytes", False) verb = maybe_unicode_to_argv(verb) args = [maybe_unicode_to_argv(a) for a in args] nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs] @@ -92,36 +99,50 @@ def run_cli_native(verb, *args, **kwargs): nodeargs=nodeargs, ) argv = nodeargs + [verb] + list(args) - if encoding is None: + stdin = kwargs.get("stdin", "") + if PY2: # The original behavior, the Python 2 behavior, is to accept either # bytes or unicode and try to automatically encode or decode as # necessary. This works okay for ASCII and if LANG is set # appropriately. These aren't great constraints so we should move # away from this behavior. + # + # The encoding attribute doesn't change StringIO behavior on Python 2, + # but it's there for realism of the emulation. + stdin = StringIO(stdin) + stdin.encoding = encoding stdout = StringIO() + stdout.encoding = encoding stderr = StringIO() - stdin = StringIO(kwargs.get("stdin", "")) + stderr.encoding = encoding else: # The new behavior, the Python 3 behavior, is to accept unicode and - # encode it using a specific encoding. For older versions of Python - # 3, the encoding is determined from LANG (bad) but for newer Python - # 3, the encoding is always utf-8 (good). Tests can pass in different - # encodings to exercise different behaviors. + # encode it using a specific encoding. For older versions of Python 3, + # the encoding is determined from LANG (bad) but for newer Python 3, + # the encoding is either LANG if it supports full Unicode, otherwise + # utf-8 (good). Tests can pass in different encodings to exercise + # different behaviors. + if isinstance(stdin, str): + stdin = stdin.encode(encoding) + stdin = TextIOWrapper(BytesIO(stdin), encoding) stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) stdin = TextIOWrapper(BytesIO(kwargs.get("stdin", b"")), encoding) d = defer.succeed(argv) d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout, stderr=stderr, stdin=stdin) - d.addCallback( - runner.dispatch, - stdin=stdin, - stdout=stdout, - stderr=stderr, - ) - def _done(rc): + d.addCallback(runner.dispatch, + stdin=stdin, + stdout=stdout, stderr=stderr) + def _done(rc, stdout=stdout, stderr=stderr): + if return_bytes and PY3: + stdout = stdout.buffer + stderr = stderr.buffer return 0, _getvalue(stdout), _getvalue(stderr) - def _err(f): + def _err(f, stdout=stdout, stderr=stderr): f.trap(SystemExit) + if return_bytes and PY3: + stdout = stdout.buffer + stderr = stderr.buffer return f.value.code, _getvalue(stdout), _getvalue(stderr) d.addCallbacks(_done, _err) return d @@ -192,7 +213,7 @@ class DevNullDictionary(dict): return def insecurerandstr(n): - return b''.join(map(bchr, map(randrange, [0]*n, [256]*n))) + return b''.join(map(bchr, list(map(randrange, [0]*n, [256]*n)))) def flip_bit(good, which): """Flip the low-order bit of good[which].""" @@ -222,9 +243,9 @@ class ReallyEqualMixin(object): # type. They're equal, and _logically_ the same type, but have # different types in practice. if a.__class__ == future_str: - a = unicode(a) + a = str(a) if b.__class__ == future_str: - b = unicode(b) + b = str(b) self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) @@ -308,7 +329,7 @@ class ShouldFailMixin(object): of the message wrapped by this Failure, or the test will fail. """ - assert substring is None or isinstance(substring, (bytes, unicode)) + assert substring is None or isinstance(substring, (bytes, str)) d = defer.maybeDeferred(callable, *args, **kwargs) def done(res): if isinstance(res, failure.Failure): @@ -399,28 +420,8 @@ class TimezoneMixin(object): return hasattr(time, 'tzset') -try: - import win32file - import win32con - def make_readonly(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY) - def make_accessible(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) -except ImportError: - import stat - def _make_readonly(path): - os.chmod(path, stat.S_IREAD) - os.chmod(os.path.dirname(path), stat.S_IREAD) - def _make_accessible(path): - os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - make_readonly = _make_readonly - make_accessible = _make_accessible - - __all__ = [ - "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin", - "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", + "TestMixin", "ShouldFailMixin", "StallMixin", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" ] diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index ce1670341..bd55a9fe9 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str __all__ = [ diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 63c24f08a..1685744fd 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -1,12 +1,21 @@ """ Tools aimed at the interaction between tests and Eliot. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # Python 2 compatibility # Can't use `builtins.str` because it's not JSON encodable: # `exceptions.TypeError: is not JSON-encodeable` from past.builtins import unicode as str from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + from six import ensure_text __all__ = [ @@ -45,7 +54,7 @@ from twisted.python.monkey import ( MonkeyPatcher, ) -from ..util.jsonbytes import BytesJSONEncoder +from ..util.jsonbytes import AnyBytesJSONEncoder _NAME = Field.for_types( @@ -67,7 +76,7 @@ RUN_TEST = ActionType( if PY2: _memory_logger = MemoryLogger else: - _memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder) + _memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder) @attr.s diff --git a/src/allmydata/test/python3_tests.py b/src/allmydata/test/python3_tests.py deleted file mode 100644 index 9326caa51..000000000 --- a/src/allmydata/test/python3_tests.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -This module defines the subset of the full test suite which is expected to -pass on Python 3 in a way which makes that suite discoverable by trial. - -This module has been ported to Python 3. -""" - -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from twisted.python.reflect import ( - namedModule, -) -from twisted.trial.runner import ( - TestLoader, -) -from twisted.trial.unittest import ( - TestSuite, -) - -from allmydata.util._python3 import ( - PORTED_TEST_MODULES, -) - -def testSuite(): - loader = TestLoader() - return TestSuite(list( - loader.loadModule(namedModule(module)) - for module - in PORTED_TEST_MODULES - )) diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py deleted file mode 100644 index 44f2123f9..000000000 --- a/src/allmydata/test/status.py +++ /dev/null @@ -1,16 +0,0 @@ - -class FakeStatus(object): - def __init__(self): - self.status = [] - - def setServiceParent(self, p): - pass - - def get_status(self): - return self.status - - def get_storage_index(self): - return None - - def get_size(self): - return None diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 17ec89078..d3f1ec7c9 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -1,8 +1,17 @@ """ A storage server plugin the test suite can use to validate the functionality. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str, native_str_to_bytes from six import ensure_str diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index d8b758dab..fea71799a 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -1,6 +1,16 @@ """ Hypothesis strategies use for testing Tahoe-LAFS. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from hypothesis.strategies import ( one_of, diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index f808f72ab..d5198d326 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -39,8 +39,10 @@ dBSD8940XU3YW+oeq8e+p3yQ2GinHfeJ3BYQyNQLuMAJ """) DUMMY_ACCOUNTS = u"""\ -alice password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 +alice herpassword URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222 + +# dennis password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333 """.format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii") @@ -54,7 +56,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase): abspath = abspath_expanduser_unicode(str(self.account_file.path)) self.checker = auth.AccountFileChecker(None, abspath) - def test_unknown_user(self): + def test_unknown_user_ssh(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object with a @@ -65,7 +67,20 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) - def test_password_auth_user(self): + def test_unknown_user_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + UnauthorizedLogin if called with an SSHPrivateKey object with a + username not present in the account file. + + We use a commented out user, so we're also checking that comments are + skipped. + """ + key_credentials = credentials.UsernamePassword(b"dennis", b"password") + d = self.checker.requestAvatarId(key_credentials) + return self.assertFailure(d, error.UnauthorizedLogin) + + def test_password_auth_user_with_ssh_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with UnauthorizedLogin if called with an SSHPrivateKey object for a username @@ -76,6 +91,43 @@ class AccountFileCheckerKeyTests(unittest.TestCase): avatarId = self.checker.requestAvatarId(key_credentials) return self.assertFailure(avatarId, error.UnauthorizedLogin) + def test_password_auth_user_with_correct_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + the user if the correct password is given. + """ + key_credentials = credentials.UsernamePassword(b"alice", b"herpassword") + d = self.checker.requestAvatarId(key_credentials) + def authenticated(avatarId): + self.assertEqual( + (b"alice", + b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), + (avatarId.username, avatarId.rootcap)) + return d + + def test_password_auth_user_with_correct_hashed_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + the user if the correct password is given in hashed form. + """ + key_credentials = credentials.UsernameHashedPassword(b"alice", b"herpassword") + d = self.checker.requestAvatarId(key_credentials) + def authenticated(avatarId): + self.assertEqual( + (b"alice", + b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"), + (avatarId.username, avatarId.rootcap)) + return d + + def test_password_auth_user_with_wrong_password(self): + """ + AccountFileChecker.requestAvatarId returns a Deferred that fires with + UnauthorizedLogin if the wrong password is given. + """ + key_credentials = credentials.UsernamePassword(b"alice", b"WRONG") + avatarId = self.checker.requestAvatarId(key_credentials) + return self.assertFailure(avatarId, error.UnauthorizedLogin) + def test_unrecognized_key(self): """ AccountFileChecker.requestAvatarId returns a Deferred that fires with diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index dc008d300..5d91b5ea4 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -418,7 +418,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): f.write("deadbeef") token = c.get_auth_token() - self.assertEqual("deadbeef", token) + self.assertEqual(b"deadbeef", token) @defer.inlineCallbacks def test_web_staticdir(self): diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py new file mode 100644 index 000000000..a689de462 --- /dev/null +++ b/src/allmydata/test/test_consumer.py @@ -0,0 +1,84 @@ +""" +Tests for allmydata.util.consumer. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from zope.interface import implementer +from twisted.trial.unittest import TestCase +from twisted.internet.interfaces import IPushProducer, IPullProducer + +from allmydata.util.consumer import MemoryConsumer + + +@implementer(IPushProducer) +@implementer(IPullProducer) +class Producer(object): + """Can be used as either streaming or non-streaming producer. + + If used as streaming, the test should call iterate() manually. + """ + + def __init__(self, consumer, data): + self.data = data + self.consumer = consumer + self.done = False + + def resumeProducing(self): + """Kick off streaming.""" + self.iterate() + + def iterate(self): + """Do another iteration of writing.""" + if self.done: + raise RuntimeError( + "There's a bug somewhere, shouldn't iterate after being done" + ) + if self.data: + self.consumer.write(self.data.pop(0)) + else: + self.done = True + self.consumer.unregisterProducer() + + +class MemoryConsumerTests(TestCase): + """Tests for MemoryConsumer.""" + + def test_push_producer(self): + """ + A MemoryConsumer accumulates all data sent by a streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) + consumer.registerProducer(producer, True) + self.assertEqual(consumer.chunks, [b"abc"]) + producer.iterate() + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, False) + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) + + def test_pull_producer(self): + """ + A MemoryConsumer accumulates all data sent by a non-streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) + consumer.registerProducer(producer, False) + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) + + +# download_to_data() is effectively tested by some of the filenode tests, e.g. +# test_immutable.py. diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index baee1acbe..652e51ea5 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -17,10 +17,10 @@ from __future__ import unicode_literals # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import PY3, PY2 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 - +from six import ensure_text import os, json from urllib.parse import quote as url_quote @@ -170,7 +170,8 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, return data def parse_streamed_json(self, s): - for unit in s.split(b"\n"): + s = ensure_text(s) + for unit in s.split("\n"): if not unit: # stream should end with a newline, so split returns "" continue @@ -746,8 +747,6 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_test_cli_good(self, ignored): d = defer.succeed(None) - if PY3: # TODO fixme once Python 3 CLI porting is done - return d d.addCallback(lambda ign: self.do_cli_manifest_stream1()) d.addCallback(lambda ign: self.do_cli_manifest_stream2()) d.addCallback(lambda ign: self.do_cli_manifest_stream3()) @@ -758,7 +757,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): return d def _check_manifest_storage_index(self, out): - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines) @@ -769,7 +768,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 8) caps = {} for l in lines: @@ -778,7 +777,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): except ValueError: cap = l.strip() path = "" - caps[cap] = path + caps[cap.encode("ascii")] = path self.failUnless(self.root.get_uri() in caps) self.failUnlessEqual(caps[self.root.get_uri()], "") self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable") @@ -814,7 +813,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_verify_cap().to_string() in lines) self.failUnless(self.mutable.get_verify_cap().to_string() in lines) @@ -827,7 +826,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split(b"\n") if l] + lines = [l.encode("utf-8") for l in out.split("\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_repair_cap().to_string() in lines) self.failUnless(self.mutable.get_repair_cap().to_string() in lines) @@ -839,7 +838,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): d = self.do_cli("stats", self.root_uri) def _check3(args): (rc, out, err) = args - lines = [l.strip() for l in out.split(b"\n") if l] + lines = [l.strip() for l in out.split("\n") if l] self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines) self.failUnless("count-literal-files: 3" in lines) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 23a0fd76b..67d331430 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1592,7 +1592,7 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation def get_write_uri(self): return self.uri.to_string() - def download_best_version(self, progress=None): + def download_best_version(self): return defer.succeed(self.data) def get_writekey(self): diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py index 3a42b0819..d61942839 100644 --- a/src/allmydata/test/test_download.py +++ b/src/allmydata/test/test_download.py @@ -1304,7 +1304,7 @@ class MyShare(object): self._dyhb_rtt = rtt def __repr__(self): - return "sh%d-on-%s" % (self._shnum, self._server.get_name()) + return "sh%d-on-%s" % (self._shnum, str(self._server.get_name(), "ascii")) class MySegmentFetcher(SegmentFetcher): def __init__(self, *args, **kwargs): @@ -1383,7 +1383,7 @@ class Selection(unittest.TestCase): self.failUnless(node.failed) self.failUnless(node.failed.check(NotEnoughSharesError)) sname = serverA.get_name() - self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused=" % sname, + self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused=" % str(sname, "ascii"), str(node.failed)) d.addCallback(_check2) return d @@ -1605,7 +1605,7 @@ class Selection(unittest.TestCase): self.failUnless(node.failed) self.failUnless(node.failed.check(NotEnoughSharesError)) sname = servers[b"peer-2"].get_name() - self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % sname, + self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % str(sname, "ascii"), str(node.failed)) d.addCallback(_check4) return d diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py index aca677323..3f915ecd2 100644 --- a/src/allmydata/test/test_eliotutil.py +++ b/src/allmydata/test/test_eliotutil.py @@ -69,7 +69,7 @@ from ..util.eliotutil import ( _parse_destination_description, _EliotLogging, ) -from ..util.jsonbytes import BytesJSONEncoder +from ..util.jsonbytes import AnyBytesJSONEncoder from .common import ( SyncTestCase, @@ -109,7 +109,7 @@ class ParseDestinationDescriptionTests(SyncTestCase): reactor = object() self.assertThat( _parse_destination_description("file:-")(reactor), - Equals(FileDestination(stdout, encoder=BytesJSONEncoder)), + Equals(FileDestination(stdout, encoder=AnyBytesJSONEncoder)), ) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index f7987d466..062c64ba1 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -379,7 +379,10 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): check(u"\n", u"\"\\x0a\"", quote_newlines=True) def test_quote_output_default(self): - self.test_quote_output_utf8(None) + """Default is the encoding of sys.stdout if known, otherwise utf-8.""" + encoding = getattr(sys.stdout, "encoding") or "utf-8" + self.assertEqual(quote_output(u"\u2621"), + quote_output(u"\u2621", encoding=encoding)) def win32_other(win32, other): diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index bb22d551f..a385abe54 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -1,4 +1,16 @@ -#!/usr/bin/python +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_binary + import os from twisted.python.filepath import FilePath @@ -43,7 +55,7 @@ class MultiIntroTests(unittest.TestCase): u'intro2':{ 'furl': 'furl4' }, }, } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) # get a client and count of introducer_clients myclient = yield create_client(self.basedir) ic_count = len(myclient.introducer_clients) @@ -73,7 +85,7 @@ class MultiIntroTests(unittest.TestCase): tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl # assertions - self.failUnlessEqual(fake_furl, tahoe_cfg_furl) + self.failUnlessEqual(fake_furl, str(tahoe_cfg_furl, "utf-8")) self.assertEqual( list( warning["message"] @@ -97,10 +109,10 @@ class MultiIntroTests(unittest.TestCase): u'default': { 'furl': 'furl1' }, }, } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) FilePath(self.basedir).child("tahoe.cfg").setContent( - "[client]\n" - "introducer.furl = furl1\n" + b"[client]\n" + b"introducer.furl = furl1\n" ) with self.assertRaises(ValueError) as ctx: @@ -112,7 +124,7 @@ class MultiIntroTests(unittest.TestCase): "please fix impossible configuration.", ) -SIMPLE_YAML = """ +SIMPLE_YAML = b""" introducers: one: furl: furl1 @@ -121,7 +133,7 @@ introducers: # this format was recommended in docs/configuration.rst in 1.12.0, but it # isn't correct (the "furl = furl1" line is recorded as the string value of # the ["one"] key, instead of being parsed as a single-key dictionary). -EQUALS_YAML = """ +EQUALS_YAML = b""" introducers: one: furl = furl1 """ @@ -147,17 +159,17 @@ class NoDefault(unittest.TestCase): connections = {'introducers': { u'one': { 'furl': 'furl1' }, }} - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) myclient = yield create_client(self.basedir) tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl - self.assertEquals(tahoe_cfg_furl, 'furl1') + self.assertEquals(tahoe_cfg_furl, b'furl1') @defer.inlineCallbacks def test_real_yaml(self): self.yaml_path.setContent(SIMPLE_YAML) myclient = yield create_client(self.basedir) tahoe_cfg_furl = myclient.introducer_clients[0].introducer_furl - self.assertEquals(tahoe_cfg_furl, 'furl1') + self.assertEquals(tahoe_cfg_furl, b'furl1') @defer.inlineCallbacks def test_invalid_equals_yaml(self): @@ -172,6 +184,6 @@ class NoDefault(unittest.TestCase): @defer.inlineCallbacks def test_introducerless(self): connections = {'introducers': {} } - self.yaml_path.setContent(yamlutil.safe_dump(connections)) + self.yaml_path.setContent(ensure_binary(yamlutil.safe_dump(connections))) myclient = yield create_client(self.basedir) self.assertEquals(len(myclient.introducer_clients), 0) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index a19da1013..ca70c4c52 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -6,7 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, native_str +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -544,6 +544,14 @@ def _stub_allocate_tcp_port(): """ return 999 +def _stub_none(): + """ + A function like ``_stub_allocate_tcp`` or ``_stub_get_local_addresses_sync`` + but that return an empty list since ``allmydata.node._tub_portlocation`` requires a + callable for paramter 1 and 2 counting from 0. + """ + return [] + class TestMissingPorts(unittest.TestCase): """ @@ -564,7 +572,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) + _tub_portlocation(config, _stub_none, _stub_none) def test_listen_on_zero_with_host(self): """ @@ -577,10 +585,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( # type: ignore - "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" - ) + _tub_portlocation(config, _stub_none, _stub_none) def test_parsing_tcp(self): """ diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index fc9ebe17a..c641d2dba 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -2,6 +2,16 @@ Tests to check for Python2 regressions """ +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from unittest import skipUnless from inspect import isclass from twisted.python.modules import getModule @@ -15,7 +25,6 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.scripts.types_", - "allmydata.test.check_load", "allmydata.test._win_subprocess", "allmydata.windows.registry", "allmydata.windows.fixups", @@ -38,10 +47,12 @@ def defined_here(cls, where): """ return cls.__module__ == where + class PythonTwoRegressions(TestCase): """ Regression tests for Python 2 behaviors related to Python 3 porting. """ + @skipUnless(PY2, "No point in running on Python 3.") def test_new_style_classes(self): """ All classes in Tahoe-LAFS are new-style. diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py deleted file mode 100644 index c1f0e83d6..000000000 --- a/src/allmydata/test/test_python3.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Tests related to the Python 3 porting effort itself. - -This module has been ported to Python 3. -""" -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2, native_str -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -from twisted.python.modules import ( - getModule, -) -from twisted.trial.unittest import ( - SynchronousTestCase, -) - -from allmydata.util._python3 import PORTED_MODULES, PORTED_TEST_MODULES - - -class Python3PortingEffortTests(SynchronousTestCase): - - def test_finished_porting(self): - """ - Tahoe-LAFS has been ported to Python 3. - - Once - https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 - is completed this test should pass (and can be deleted!). - """ - tahoe_lafs_module_names = set(all_module_names("allmydata")) - ported_names = set(ported_module_names()) - self.assertEqual( - tahoe_lafs_module_names - ported_names, - set(), - "Some unported modules remain: {}".format( - unported_report( - tahoe_lafs_module_names, - ported_names, - ), - ), - ) - test_finished_porting.todo = native_str( # type: ignore - "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", - ) - - def test_ported_modules_exist(self): - """ - All modules listed as ported exist and belong to Tahoe-LAFS. - """ - tahoe_lafs_module_names = set(all_module_names("allmydata")) - ported_names = set(ported_module_names()) - unknown = ported_names - tahoe_lafs_module_names - self.assertEqual( - unknown, - set(), - "Some supposedly-ported modules weren't found: {}.".format(sorted(unknown)), - ) - - def test_ported_modules_distinct(self): - """ - The ported modules list doesn't contain duplicates. - """ - ported_names_list = ported_module_names() - ported_names_list.sort() - ported_names_set = set(ported_names_list) - ported_names_unique_list = list(ported_names_set) - ported_names_unique_list.sort() - self.assertEqual( - ported_names_list, - ported_names_unique_list, - ) - - -def all_module_names(toplevel): - """ - :param unicode toplevel: The name of a top-level Python package. - - :return iterator[unicode]: An iterator of ``unicode`` giving the names of - all modules within the given top-level Python package. - """ - allmydata = getModule(toplevel) - for module in allmydata.walkModules(): - name = module.name - if PY2: - name = name.decode("utf-8") - yield name - - -def ported_module_names(): - """ - :return list[unicode]: A ``list`` of ``unicode`` giving the names of - Tahoe-LAFS modules which have been ported to Python 3. - """ - return PORTED_MODULES + PORTED_TEST_MODULES - - -def unported_report(tahoe_lafs_module_names, ported_names): - return """ -Ported files: {} / {} -Ported lines: {} / {} -""".format( - len(ported_names), - len(tahoe_lafs_module_names), - sum(map(count_lines, ported_names)), - sum(map(count_lines, tahoe_lafs_module_names)), -) - -def count_lines(module_name): - module = getModule(module_name) - try: - source = module.filePath.getContent() - except Exception as e: - print((module_name, e)) - return 0 - lines = source.splitlines() - nonblank = [_f for _f in lines if _f] - return len(nonblank) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f6a7c2ee1..a9ec83524 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -17,6 +17,7 @@ from six import ensure_text import os.path, re, sys from os import linesep +import locale from eliot import ( log_call, @@ -88,12 +89,16 @@ def run_bintahoe(extra_argv, python_options=None): argv = [executable] if python_options is not None: argv.extend(python_options) - argv.extend([u"-m", u"allmydata.scripts.runner"]) + argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"]) argv.extend(extra_argv) argv = list(unicode_to_argv(arg) for arg in argv) p = Popen(argv, stdout=PIPE, stderr=PIPE) - out = p.stdout.read().decode("utf-8") - err = p.stderr.read().decode("utf-8") + if PY2: + encoding = "utf-8" + else: + encoding = locale.getpreferredencoding(False) + out = p.stdout.read().decode(encoding) + err = p.stderr.read().decode(encoding) returncode = p.wait() return (out, err, returncode) @@ -103,7 +108,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): """ The runner script receives unmangled non-ASCII values in argv. """ - tricky = u"\u2621" + tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) self.assertEqual(returncode, 1) self.assertIn(u"Unknown command: " + tricky, out) @@ -515,7 +520,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): 0, "Expected error message from '{}', got something else: {}".format( description, - p.get_buffered_output(), + str(p.get_buffered_output(), "utf-8"), ), ) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index ce575ce7a..627b6ef29 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,12 +1,12 @@ """ -Ported to Python 3, partially: test_filesystem* will be done in a future round. +Ported to Python 3. """ from __future__ import print_function from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: # Don't import bytes since it causes issues on (so far unported) modules on Python 2. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 @@ -16,7 +16,6 @@ from six import ensure_text, ensure_str import os, re, sys, time, json from functools import partial -from unittest import skipIf from bs4 import BeautifulSoup @@ -44,14 +43,14 @@ from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData +from allmydata.scripts.runner import PYTHON_3_WARNING from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure from twisted.python.filepath import ( FilePath, ) - -from ._twisted_9607 import ( +from twisted.internet.utils import ( getProcessOutputAndValue, ) @@ -77,7 +76,7 @@ class RunBinTahoeMixin(object): # support env yet and is also synchronous. If we could get rid of # this in favor of that, though, it would probably be an improvement. command = sys.executable - argv = python_options + ["-m", "allmydata.scripts.runner"] + args + argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args if env is None: env = os.environ @@ -1666,9 +1665,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) d.addCallback(self._test_control) - if PY2: - # TODO when CLI is ported to Python 3, reenable. - d.addCallback(self._test_cli) + d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data # P/s2-ro/ @@ -2299,7 +2296,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_aliases_1(out_and_err): (out, err) = out_and_err self.failUnlessEqual(err, "") - self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % private_uri) + self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % str(private_uri, "ascii")) d.addCallback(_check_aliases_1) # now that that's out of the way, remove root_dir.cap and work with @@ -2356,7 +2353,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): (out, err) = out_and_err self.failUnlessEqual(err, "") if filenum is not None: - self.failUnlessEqual(out, datas[filenum]) + self.failUnlessEqual(out, str(datas[filenum], "ascii")) if data is not None: self.failUnlessEqual(out, data) @@ -2370,7 +2367,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): uri0 = out.strip() return run(None, "get", uri0) d.addCallback(_put_out) - d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], datas[0])) + d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], str(datas[0], "ascii"))) d.addCallback(run, "put", files[1], "subdir/tahoe-file1") # tahoe put bar tahoe:FOO @@ -2412,14 +2409,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_outfile0(out_and_err): (out, err) = out_and_err data = open(outfile0,"rb").read() - self.failUnlessEqual(data, "data to be uploaded: file2\n") + self.failUnlessEqual(data, b"data to be uploaded: file2\n") d.addCallback(_check_outfile0) outfile1 = os.path.join(self.basedir, "outfile0") d.addCallback(run, "get", "tahoe:subdir/tahoe-file1", outfile1) def _check_outfile1(out_and_err): (out, err) = out_and_err data = open(outfile1,"rb").read() - self.failUnlessEqual(data, "data to be uploaded: file1\n") + self.failUnlessEqual(data, b"data to be uploaded: file1\n") d.addCallback(_check_outfile1) d.addCallback(run, "unlink", "tahoe-file0") @@ -2456,7 +2453,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if "file3" in l: rw_uri = self._mutable_file3_uri u = uri.from_string_mutable_filenode(rw_uri) - ro_uri = u.get_readonly().to_string() + ro_uri = str(u.get_readonly().to_string(), "ascii") self.failUnless(ro_uri in l) d.addCallback(_check_ls_rouri) @@ -2529,17 +2526,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): dn = os.path.join(self.basedir, "dir1") os.makedirs(dn) with open(os.path.join(dn, "rfile1"), "wb") as f: - f.write("rfile1") + f.write(b"rfile1") with open(os.path.join(dn, "rfile2"), "wb") as f: - f.write("rfile2") + f.write(b"rfile2") with open(os.path.join(dn, "rfile3"), "wb") as f: - f.write("rfile3") + f.write(b"rfile3") sdn2 = os.path.join(dn, "subdir2") os.makedirs(sdn2) with open(os.path.join(sdn2, "rfile4"), "wb") as f: - f.write("rfile4") + f.write(b"rfile4") with open(os.path.join(sdn2, "rfile5"), "wb") as f: - f.write("rfile5") + f.write(b"rfile5") # from disk into tahoe d.addCallback(run, "cp", "-r", dn, "tahoe:") @@ -2583,7 +2580,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): (out, err) = out_and_err x = open(os.path.join(dn_copy2, "dir1", "subdir2", "rfile4")).read() y = uri.from_string_filenode(x) - self.failUnlessEqual(y.data, "rfile4") + self.failUnlessEqual(y.data, b"rfile4") d.addCallback(_check_capsonly) # and tahoe-to-tahoe @@ -2616,7 +2613,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d - @skipIf(PY3, "Python 3 CLI support hasn't happened yet.") def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. @@ -2640,7 +2636,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) if check_stderr: - self.failUnlessEqual(err, b"") + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) d.addCallback(_run_in_subprocess, "create-alias", "newalias") d.addCallback(_check_succeeded) @@ -2660,9 +2656,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.failUnlessEqual(err, "", str(res)) - self.failUnlessIn("tahoe-moved", out) - self.failIfIn("tahoe-file", out) + self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) + self.failUnlessIn(b"tahoe-moved", out) + self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) return d diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py index 148d813f5..86d54803a 100644 --- a/src/allmydata/test/test_tor_provider.py +++ b/src/allmydata/test/test_tor_provider.py @@ -14,6 +14,7 @@ import os from twisted.trial import unittest from twisted.internet import defer, error from six.moves import StringIO +from six import ensure_str import mock from ..util import tor_provider from ..scripts import create_node, runner @@ -185,7 +186,8 @@ class CreateOnion(unittest.TestCase): protocol))) txtorcon = mock.Mock() ehs = mock.Mock() - ehs.private_key = b"privkey" + # This appears to be a native string in the real txtorcon object... + ehs.private_key = ensure_str("privkey") ehs.hostname = "ONION.onion" txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs) ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None)) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 5f5db82bd..a03845ed6 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -17,15 +17,17 @@ import yaml import json from twisted.trial import unittest +from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import jsonbytes from allmydata.util import pollmixin from allmydata.util import yamlutil +from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin - +from .no_network import fireNow, LocalWrapper if six.PY3: long = int @@ -480,7 +482,12 @@ class EqButNotIs(object): class YAML(unittest.TestCase): def test_convert(self): - data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) + """ + Unicode and (ASCII) native strings get roundtripped to Unicode strings. + """ + data = yaml.safe_dump( + [six.ensure_str("str"), u"unicode", u"\u1234nicode"] + ) back = yamlutil.safe_load(data) self.assertIsInstance(back[0], str) self.assertIsInstance(back[1], str) @@ -488,10 +495,10 @@ class YAML(unittest.TestCase): class JSONBytes(unittest.TestCase): - """Tests for BytesJSONEncoder.""" + """Tests for jsonbytes module.""" def test_encode_bytes(self): - """BytesJSONEncoder can encode bytes. + """jsonbytes.dumps() encodes bytes. Bytes are presumed to be UTF-8 encoded. """ @@ -507,11 +514,82 @@ class JSONBytes(unittest.TestCase): self.assertEqual(json.loads(encoded), expected) self.assertEqual(jsonbytes.loads(encoded), expected) - def test_encode_unicode(self): - """BytesJSONEncoder encodes Unicode string as usual.""" + """jsonbytes.dumps() encodes Unicode string as usual.""" expected = { u"hello": [1, u"cd"], } encoded = jsonbytes.dumps(expected) self.assertEqual(json.loads(encoded), expected) + + def test_dumps_bytes(self): + """jsonbytes.dumps_bytes always returns bytes.""" + x = {u"def\N{SNOWMAN}\uFF00": 123} + encoded = jsonbytes.dumps_bytes(x) + self.assertIsInstance(encoded, bytes) + self.assertEqual(json.loads(encoded), x) + + def test_any_bytes_unsupported_by_default(self): + """By default non-UTF-8 bytes raise error.""" + bytestring = b"abc\xff\x00" + with self.assertRaises(UnicodeDecodeError): + jsonbytes.dumps(bytestring) + with self.assertRaises(UnicodeDecodeError): + jsonbytes.dumps_bytes(bytestring) + with self.assertRaises(UnicodeDecodeError): + json.dumps(bytestring, cls=jsonbytes.UTF8BytesJSONEncoder) + + def test_any_bytes(self): + """If any_bytes is True, non-UTF-8 bytes don't break encoding.""" + bytestring = b"abc\xff\xff123" + o = {bytestring: bytestring} + expected = {"abc\\xff\\xff123": "abc\\xff\\xff123"} + self.assertEqual( + json.loads(jsonbytes.dumps(o, any_bytes=True)), + expected, + ) + self.assertEqual( + json.loads(json.dumps( + o, cls=jsonbytes.AnyBytesJSONEncoder)), + expected, + ) + self.assertEqual( + json.loads(jsonbytes.dumps(o, any_bytes=True)), + expected + ) + + + +class FakeGetVersion(object): + """Emulate an object with a get_version.""" + + def __init__(self, result): + self.result = result + + def remote_get_version(self): + if isinstance(self.result, Exception): + raise self.result + return self.result + + +class RrefUtilTests(unittest.TestCase): + """Tests for rrefutil.""" + + def test_version_returned(self): + """If get_version() succeeded, it is set on the rref.""" + rref = LocalWrapper(FakeGetVersion(12345), fireNow) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "default") + ) + self.assertEqual(result.version, 12345) + self.assertIdentical(result, rref) + + def test_exceptions(self): + """If get_version() failed, default version is set on the rref.""" + for exception in (Violation(), RemoteException(ValueError())): + rref = LocalWrapper(FakeGetVersion(exception), fireNow) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "Default") + ) + self.assertEqual(result.version, "Default") + self.assertIdentical(result, rref) diff --git a/src/allmydata/test/test_windows.py b/src/allmydata/test/test_windows.py index 01e4a57c1..bae56bfed 100644 --- a/src/allmydata/test/test_windows.py +++ b/src/allmydata/test/test_windows.py @@ -79,6 +79,7 @@ slow_settings = settings( ) @skipUnless(platform.isWindows(), "get_argv is Windows-only") +@skipUnless(PY2, "Not used on Python 3.") class GetArgvTests(SyncTestCase): """ Tests for ``get_argv``. @@ -172,6 +173,7 @@ class GetArgvTests(SyncTestCase): @skipUnless(platform.isWindows(), "intended for Windows-only codepaths") +@skipUnless(PY2, "Not used on Python 3.") class UnicodeOutputTests(SyncTestCase): """ Tests for writing unicode to stdout and stderr. diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 00a40e3c5..43a13a902 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index 99c91ef5c..f764da79d 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import attr from testtools.matchers import Mismatch diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index ef2718df4..edcf32268 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -660,7 +660,6 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if line] except ValueError: print("response is:", res) - print("undecodeable line was '%s'" % line) raise self.failUnlessReallyEqual(len(units), 5+1) # should be parent-first diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 08d95bda9..ba0a5beb9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -15,9 +15,14 @@ from os.path import join from bs4 import BeautifulSoup -from twisted.trial import unittest from twisted.internet import reactor from twisted.internet import defer +from testtools.twistedsupport import succeeded + +from ..common import ( + SyncTestCase, + AsyncTestCase, +) from foolscap.api import ( fireEventually, @@ -53,6 +58,11 @@ from ..common_web import ( render, ) +from testtools.matchers import ( + Equals, + AfterPreprocessing, +) + @defer.inlineCallbacks def create_introducer_webish(reactor, port_assigner, basedir): @@ -86,11 +96,10 @@ def create_introducer_webish(reactor, port_assigner, basedir): yield fireEventually(None) intro_node.startService() - defer.returnValue((intro_node, ws)) -class IntroducerWeb(unittest.TestCase): +class IntroducerWeb(AsyncTestCase): """ Tests for web-facing functionality of an introducer node. """ @@ -102,6 +111,7 @@ class IntroducerWeb(unittest.TestCase): # Anything using Foolscap leaves some timer trash in the reactor that # we have to arrange to have cleaned up. self.addCleanup(lambda: flushEventualQueue(None)) + return super(IntroducerWeb, self).setUp() @defer.inlineCallbacks def test_welcome(self): @@ -187,7 +197,7 @@ class IntroducerWeb(unittest.TestCase): self.assertEqual(data["announcement_summary"], {}) -class IntroducerRootTests(unittest.TestCase): +class IntroducerRootTests(SyncTestCase): """ Tests for ``IntroducerRoot``. """ @@ -223,15 +233,11 @@ class IntroducerRootTests(unittest.TestCase): ) resource = IntroducerRoot(introducer_node) - response = json.loads( - self.successResultOf( - render(resource, {b"t": [b"json"]}), - ), - ) - self.assertEqual( + response = render(resource, {b"t": [b"json"]}) + expected = { + u"subscription_summary": {"arbitrary": 2}, + u"announcement_summary": {"arbitrary": 1}, + } + self.assertThat( response, - { - u"subscription_summary": {"arbitrary": 2}, - u"announcement_summary": {"arbitrary": 1}, - }, - ) + succeeded(AfterPreprocessing(json.loads, Equals(expected)))) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 5d697f910..89ec7ba42 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -92,7 +92,7 @@ class TestStreamingLogs(unittest.TestCase): @inlineCallbacks def test_one_log(self): """ - write a single Eliot log and see it streamed via websocket + Write a single Eliot log action and see it streamed via websocket. """ proto = yield self.agent.open( @@ -106,14 +106,18 @@ class TestStreamingLogs(unittest.TestCase): proto.on("message", got_message) @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): + def do_a_thing(arguments): pass - do_a_thing() + do_a_thing(arguments=[u"hello", b"good-\xff-day", 123, {"a": 35}, [None]]) proto.transport.loseConnection() yield proto.is_closed self.assertEqual(len(messages), 2) + self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action") + self.assertEqual(messages[0]["arguments"], + ["hello", "good-\\xff-day", 123, {"a": 35}, [None]]) + self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action") self.assertEqual("started", messages[0]["action_status"]) self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 6b25305c6..1c9d6b65c 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -218,7 +218,7 @@ class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/35 return self.connected def get_version(self): return { - "application-version": "1.0" + b"application-version": b"1.0" } def get_permutation_seed(self): return b"" @@ -1394,8 +1394,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(self.BAR_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(self.BAR_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got) @@ -3015,8 +3015,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got_headers(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(NEW2_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(NEW2_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got_headers) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 70742b7b2..5641771d3 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -16,7 +16,7 @@ if PY2: # Don't import bytes or str, to prevent future's newbytes leaking and # breaking code that only expects normal bytes. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401 - str = unicode + from past.builtins import unicode as str from past.builtins import unicode, long diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py deleted file mode 100644 index 5478245ae..000000000 --- a/src/allmydata/util/_python3.py +++ /dev/null @@ -1,243 +0,0 @@ -""" -Track the port to Python 3. - -The two easiest ways to run the part of the test suite which is expected to -pass on Python 3 are:: - - $ tox -e py36 - -and:: - - $ trial allmydata.test.python3_tests - -This module has been ported to Python 3. -""" - -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -# Keep these sorted alphabetically, to reduce merge conflicts: -PORTED_MODULES = [ - "allmydata.__main__", - "allmydata._auto_deps", - "allmydata._monkeypatch", - "allmydata.blacklist", - "allmydata.check_results", - "allmydata.client", - "allmydata.codec", - "allmydata.control", - "allmydata.crypto", - "allmydata.crypto.aes", - "allmydata.crypto.ed25519", - "allmydata.crypto.error", - "allmydata.crypto.rsa", - "allmydata.crypto.util", - "allmydata.deep_stats", - "allmydata.dirnode", - "allmydata.frontends.sftpd", - "allmydata.hashtree", - "allmydata.history", - "allmydata.immutable.checker", - "allmydata.immutable.downloader", - "allmydata.immutable.downloader.common", - "allmydata.immutable.downloader.fetcher", - "allmydata.immutable.downloader.finder", - "allmydata.immutable.downloader.node", - "allmydata.immutable.downloader.segmentation", - "allmydata.immutable.downloader.share", - "allmydata.immutable.downloader.status", - "allmydata.immutable.encode", - "allmydata.immutable.filenode", - "allmydata.immutable.happiness_upload", - "allmydata.immutable.layout", - "allmydata.immutable.literal", - "allmydata.immutable.offloaded", - "allmydata.immutable.repairer", - "allmydata.immutable.upload", - "allmydata.interfaces", - "allmydata.introducer.client", - "allmydata.introducer.common", - "allmydata.introducer.interfaces", - "allmydata.introducer.server", - "allmydata.monitor", - "allmydata.mutable.checker", - "allmydata.mutable.common", - "allmydata.mutable.filenode", - "allmydata.mutable.layout", - "allmydata.mutable.publish", - "allmydata.mutable.repairer", - "allmydata.mutable.retrieve", - "allmydata.mutable.servermap", - "allmydata.node", - "allmydata.nodemaker", - "allmydata.scripts.create_node", - "allmydata.scripts.types_", - "allmydata.stats", - "allmydata.storage_client", - "allmydata.storage.common", - "allmydata.storage.crawler", - "allmydata.storage.expirer", - "allmydata.storage.immutable", - "allmydata.storage.lease", - "allmydata.storage.mutable", - "allmydata.storage.server", - "allmydata.storage.shares", - "allmydata.test.no_network", - "allmydata.test.matchers", - "allmydata.test.mutable.util", - "allmydata.testing.web", - "allmydata.unknown", - "allmydata.uri", - "allmydata.util._python3", - "allmydata.util.abbreviate", - "allmydata.util.assertutil", - "allmydata.util.base32", - "allmydata.util.base62", - "allmydata.util.configutil", - "allmydata.util.connection_status", - "allmydata.util.deferredutil", - "allmydata.util.dictutil", - "allmydata.util.eliotutil", - "allmydata.util.encodingutil", - "allmydata.util.fileutil", - "allmydata.util.gcutil", - "allmydata.util.happinessutil", - "allmydata.util.hashutil", - "allmydata.util.humanreadable", - "allmydata.util.i2p_provider", - "allmydata.util.idlib", - "allmydata.util.iputil", - "allmydata.util.jsonbytes", - "allmydata.util.log", - "allmydata.util.mathutil", - "allmydata.util.namespace", - "allmydata.util.netstring", - "allmydata.util.observer", - "allmydata.util.pipeline", - "allmydata.util.pollmixin", - "allmydata.util.spans", - "allmydata.util.statistics", - "allmydata.util.time_format", - "allmydata.util.tor_provider", - "allmydata.web.check_results", - "allmydata.web.common", - "allmydata.web.directory", - "allmydata.web.filenode", - "allmydata.web.info", - "allmydata.web.introweb", - "allmydata.web.logs", - "allmydata.web.operations", - "allmydata.web.private", - "allmydata.web.root", - "allmydata.web.status", - "allmydata.web.storage", - "allmydata.web.storage_plugins", - "allmydata.web.unlinked", - "allmydata.webish", -] - -PORTED_TEST_MODULES = [ - "allmydata.test.cli.test_alias", - "allmydata.test.cli.test_backupdb", - "allmydata.test.cli.test_create", - "allmydata.test.cli.test_status", - - "allmydata.test.mutable.test_checker", - "allmydata.test.mutable.test_datahandle", - "allmydata.test.mutable.test_different_encoding", - "allmydata.test.mutable.test_exceptions", - "allmydata.test.mutable.test_filehandle", - "allmydata.test.mutable.test_filenode", - "allmydata.test.mutable.test_interoperability", - "allmydata.test.mutable.test_multiple_encodings", - "allmydata.test.mutable.test_multiple_versions", - "allmydata.test.mutable.test_problems", - "allmydata.test.mutable.test_repair", - "allmydata.test.mutable.test_roundtrip", - "allmydata.test.mutable.test_servermap", - "allmydata.test.mutable.test_update", - "allmydata.test.mutable.test_version", - "allmydata.test.test_abbreviate", - "allmydata.test.test_auth", - "allmydata.test.test_base32", - "allmydata.test.test_base62", - "allmydata.test.test_checker", - "allmydata.test.test_client", - "allmydata.test.test_codec", - "allmydata.test.test_common_util", - "allmydata.test.test_configutil", - "allmydata.test.test_connections", - "allmydata.test.test_connection_status", - "allmydata.test.test_crawler", - "allmydata.test.test_crypto", - - # Only partially ported, CLI-using test code is disabled for now until CLI - # is ported. - "allmydata.test.test_deepcheck", - - "allmydata.test.test_deferredutil", - "allmydata.test.test_dictutil", - "allmydata.test.test_dirnode", - "allmydata.test.test_download", - "allmydata.test.test_eliotutil", - "allmydata.test.test_encode", - "allmydata.test.test_encodingutil", - "allmydata.test.test_filenode", - "allmydata.test.test_happiness", - "allmydata.test.test_hashtree", - "allmydata.test.test_hashutil", - "allmydata.test.test_helper", - "allmydata.test.test_humanreadable", - "allmydata.test.test_hung_server", - "allmydata.test.test_i2p_provider", - "allmydata.test.test_immutable", - "allmydata.test.test_introducer", - "allmydata.test.test_iputil", - "allmydata.test.test_json_metadata", - "allmydata.test.test_log", - "allmydata.test.test_monitor", - "allmydata.test.test_netstring", - "allmydata.test.test_no_network", - "allmydata.test.test_node", - "allmydata.test.test_observer", - "allmydata.test.test_pipeline", - "allmydata.test.test_python3", - "allmydata.test.test_repairer", - "allmydata.test.test_runner", - "allmydata.test.test_sftp", - "allmydata.test.test_spans", - "allmydata.test.test_statistics", - "allmydata.test.test_stats", - "allmydata.test.test_storage", - "allmydata.test.test_storage_client", - "allmydata.test.test_storage_web", - - # Only partially ported, test_filesystem_with_cli_in_subprocess isn't - # ported yet, nor is part of test_filesystem (the call to _test_cli). This - # should be done once CLI is ported. - "allmydata.test.test_system", - - "allmydata.test.test_testing", - "allmydata.test.test_time_format", - "allmydata.test.test_tor_provider", - "allmydata.test.test_upload", - "allmydata.test.test_uri", - "allmydata.test.test_util", - "allmydata.test.web.test_common", - "allmydata.test.web.test_grid", - "allmydata.test.web.test_introducer", - "allmydata.test.web.test_logs", - "allmydata.test.web.test_private", - "allmydata.test.web.test_root", - "allmydata.test.web.test_status", - "allmydata.test.web.test_util", - "allmydata.test.web.test_web", - "allmydata.test.web.test_webish", - "allmydata.test.test_windows", -] diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index 393d7dec5..3de82974d 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -1,18 +1,28 @@ - -"""This file defines a basic download-to-memory consumer, suitable for use in -a filenode's read() method. See download_to_data() for an example of its use. """ +This file defines a basic download-to-memory consumer, suitable for use in +a filenode's read() method. See download_to_data() for an example of its use. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet.interfaces import IConsumer + @implementer(IConsumer) class MemoryConsumer(object): - def __init__(self, progress=None): + def __init__(self): self.chunks = [] self.done = False - self._progress = progress def registerProducer(self, p, streaming): self.producer = p @@ -25,16 +35,15 @@ class MemoryConsumer(object): def write(self, data): self.chunks.append(data) - if self._progress is not None: - self._progress.set_progress(sum([len(c) for c in self.chunks])) def unregisterProducer(self): self.done = True -def download_to_data(n, offset=0, size=None, progress=None): + +def download_to_data(n, offset=0, size=None): """ - :param progress: None or an IProgress implementer + Return Deferred that fires with results of reading from the given filenode. """ - d = n.read(MemoryConsumer(progress=progress), offset, size) + d = n.read(MemoryConsumer(), offset, size) d.addCallback(lambda mc: b"".join(mc.chunks)) return d diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 543dd2797..916382972 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -1,9 +1,23 @@ +""" +SQLite3 utilities. + +Test coverage currently provided by test_backupdb.py. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, sys import sqlite3 -from sqlite3 import IntegrityError -[IntegrityError] class DBError(Exception): @@ -12,7 +26,7 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, create_version=(None, None), updaters={}, just_create=False, dbname="db", - journal_mode=None, synchronous=None): + ): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. Updaters is a {newver: commands} mapping, where e.g. updaters[2] is used @@ -32,12 +46,6 @@ def get_db(dbfile, stderr=sys.stderr, # The default is unspecified according to . c.execute("PRAGMA foreign_keys = ON;") - if journal_mode is not None: - c.execute("PRAGMA journal_mode = %s;" % (journal_mode,)) - - if synchronous is not None: - c.execute("PRAGMA synchronous = %s;" % (synchronous,)) - if must_create: c.executescript(schema) c.execute("INSERT INTO version (version) VALUES (?)", (target_version,)) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 5d144eb1d..4e48fbb9f 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -87,7 +87,7 @@ from twisted.internet.defer import ( ) from twisted.application.service import Service -from .jsonbytes import BytesJSONEncoder +from .jsonbytes import AnyBytesJSONEncoder def validateInstanceOf(t): @@ -306,7 +306,7 @@ class _DestinationParser(object): rotateLength=rotate_length, maxRotatedFiles=max_rotated_files, ) - return lambda reactor: FileDestination(get_file(), BytesJSONEncoder) + return lambda reactor: FileDestination(get_file(), AnyBytesJSONEncoder) _parse_destination_description = _DestinationParser().parse @@ -333,4 +333,4 @@ def log_call_deferred(action_type): if PY2: capture_logging = eliot_capture_logging else: - capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder) + capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 637374064..576091539 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -256,7 +256,11 @@ def quote_output_u(*args, **kwargs): result = quote_output(*args, **kwargs) if isinstance(result, unicode): return result - return result.decode(kwargs.get("encoding", None) or io_encoding) + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + return result.decode(kwargs.get("encoding", None) or + getattr(sys.stdout, "encoding") or io_encoding) def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): @@ -276,7 +280,10 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): On Python 3, returns Unicode strings. """ precondition(isinstance(s, (bytes, unicode)), s) - encoding = encoding or io_encoding + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + encoding = encoding or getattr(sys.stdout, "encoding") or io_encoding if quote_newlines is None: quote_newlines = quotemarks @@ -284,7 +291,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): def _encode(s): if isinstance(s, bytes): try: - s = s.decode('utf-8') + s = s.decode("utf-8") except UnicodeDecodeError: return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py index 935187d29..08e0cb68e 100644 --- a/src/allmydata/util/jsonbytes.py +++ b/src/allmydata/util/jsonbytes.py @@ -9,46 +9,109 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import json +import codecs + +if PY2: + def backslashreplace_py2(ex): + """ + On Python 2 'backslashreplace' error handler doesn't work, so write our + own. + """ + return ''.join('\\x{:02x}'.format(ord(c)) + for c in ex.object[ex.start:ex.end]), ex.end + + codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2) -def _bytes_to_unicode(obj): - """Convert any bytes objects to unicode, recursively.""" - if isinstance(obj, bytes): - return obj.decode("utf-8") - if isinstance(obj, dict): - new_obj = {} - for k, v in obj.items(): - if isinstance(k, bytes): - k = k.decode("utf-8") - v = _bytes_to_unicode(v) - new_obj[k] = v - return new_obj - if isinstance(obj, (list, set, tuple)): - return [_bytes_to_unicode(i) for i in obj] - return obj +def bytes_to_unicode(any_bytes, obj): + """Convert bytes to unicode. - -class BytesJSONEncoder(json.JSONEncoder): + :param any_bytes: If True, also support non-UTF-8-encoded bytes. + :param obj: Object to de-byte-ify. """ - A JSON encoder than can also encode bytes. + errors = "backslashreplace" if any_bytes else "strict" + if PY2 and errors == "backslashreplace": + errors = "backslashreplace_tahoe_py2" - The bytes are assumed to be UTF-8 encoded Unicode strings. + def doit(obj): + """Convert any bytes objects to unicode, recursively.""" + if isinstance(obj, bytes): + return obj.decode("utf-8", errors=errors) + if isinstance(obj, dict): + new_obj = {} + for k, v in obj.items(): + if isinstance(k, bytes): + k = k.decode("utf-8", errors=errors) + v = doit(v) + new_obj[k] = v + return new_obj + if isinstance(obj, (list, set, tuple)): + return [doit(i) for i in obj] + return obj + + return doit(obj) + + +class UTF8BytesJSONEncoder(json.JSONEncoder): """ + A JSON encoder than can also encode UTF-8 encoded strings. + """ + def encode(self, o, **kwargs): + return json.JSONEncoder.encode( + self, bytes_to_unicode(False, o), **kwargs) + def iterencode(self, o, **kwargs): - return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs) + return json.JSONEncoder.iterencode( + self, bytes_to_unicode(False, o), **kwargs) + + +class AnyBytesJSONEncoder(json.JSONEncoder): + """ + A JSON encoder than can also encode bytes of any sort. + + Bytes are decoded to strings using UTF-8, if that fails to decode then the + bytes are quoted. + """ + def encode(self, o, **kwargs): + return json.JSONEncoder.encode( + self, bytes_to_unicode(True, o), **kwargs) + + def iterencode(self, o, **kwargs): + return json.JSONEncoder.iterencode( + self, bytes_to_unicode(True, o), **kwargs) def dumps(obj, *args, **kwargs): """Encode to JSON, supporting bytes as keys or values. - The bytes are assumed to be UTF-8 encoded Unicode strings. + :param bool any_bytes: If False (the default) the bytes are assumed to be + UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for + human consumption. """ - return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs) + any_bytes = kwargs.pop("any_bytes", False) + if any_bytes: + cls = AnyBytesJSONEncoder + else: + cls = UTF8BytesJSONEncoder + return json.dumps(obj, cls=cls, *args, **kwargs) + + +def dumps_bytes(obj, *args, **kwargs): + """Encode to JSON, then encode as bytes. + + :param bool any_bytes: If False (the default) the bytes are assumed to be + UTF-8 encoded Unicode strings. If True, non-UTF-8 bytes are quoted for + human consumption. + """ + result = dumps(obj, *args, **kwargs) + if PY3: + result = result.encode("utf-8") + return result # To make this module drop-in compatible with json module: diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 509deb6a4..b442d30bb 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -18,6 +18,16 @@ from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log +if PY2: + def bytes_to_unicode(ign, obj): + return obj +else: + # We want to convert bytes keys to Unicode, otherwise JSON serialization + # inside foolscap will fail (for details see + # https://github.com/warner/foolscap/issues/88) + from .jsonbytes import bytes_to_unicode + + NOISY = log.NOISY # 10 OPERATIONAL = log.OPERATIONAL # 20 UNUSUAL = log.UNUSUAL # 23 @@ -28,7 +38,8 @@ SCARY = log.SCARY # 35 BAD = log.BAD # 40 -msg = log.msg +def msg(*args, **kwargs): + return log.msg(*args, **bytes_to_unicode(True, kwargs)) # If log.err() happens during a unit test, the unit test should fail. We # accomplish this by sending it to twisted.log too. When a WEIRD/SCARY/BAD @@ -39,7 +50,7 @@ def err(failure=None, _why=None, **kwargs): tw_log.err(failure, _why, **kwargs) if 'level' not in kwargs: kwargs['level'] = log.UNUSUAL - return log.err(failure, _why, **kwargs) + return log.err(failure, _why, **bytes_to_unicode(True, kwargs)) class LogMixin(object): """ I remember a msg id and a facility and pass them to log.msg() """ @@ -57,7 +68,8 @@ class LogMixin(object): if pmsgid is None: pmsgid = self._grandparentmsgid kwargs = {ensure_str(k): v for (k, v) in kwargs.items()} - msgid = log.msg(msg, facility=facility, parent=pmsgid, *args, **kwargs) + msgid = log.msg(msg, facility=facility, parent=pmsgid, *args, + **bytes_to_unicode(True, kwargs)) if self._parentmsgid is None: self._parentmsgid = msgid return msgid diff --git a/src/allmydata/util/progress.py b/src/allmydata/util/progress.py deleted file mode 100644 index 3618c88dd..000000000 --- a/src/allmydata/util/progress.py +++ /dev/null @@ -1,37 +0,0 @@ -""" -Utilities relating to computing progress information. - -Ties in with the "consumer" module also -""" - -from allmydata.interfaces import IProgress -from zope.interface import implementer - - -@implementer(IProgress) -class PercentProgress(object): - """ - Represents progress as a percentage, from 0.0 to 100.0 - """ - - def __init__(self, total_size=None): - self._value = 0.0 - self.set_progress_total(total_size) - - def set_progress(self, value): - "IProgress API" - self._value = value - - def set_progress_total(self, size): - "IProgress API" - if size is not None: - size = float(size) - self._total_size = size - - @property - def progress(self): - if self._total_size is None: - return 0 # or 1.0? - if self._total_size <= 0.0: - return 0 - return (self._value / self._total_size) * 100.0 diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 40e921507..f39890ff1 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,6 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from twisted.internet import address -from foolscap.api import Violation, RemoteException, SturdyRef +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from foolscap.api import Violation, RemoteException def add_version_to_remote_reference(rref, default): @@ -18,24 +28,3 @@ def add_version_to_remote_reference(rref, default): return rref d.addCallbacks(_got_version, _no_get_version) return d - - -def connection_hints_for_furl(furl): - hints = [] - for h in SturdyRef(furl).locationHints: - # Foolscap-0.2.5 and earlier used strings in .locationHints, 0.2.6 - # through 0.6.4 used tuples of ("ipv4",host,port), 0.6.5 through - # 0.8.0 used tuples of ("tcp",host,port), and >=0.9.0 uses strings - # again. Tolerate them all. - if isinstance(h, tuple): - hints.append(":".join([str(s) for s in h])) - else: - hints.append(h) - return hints - -def stringify_remote_address(rref): - remote = rref.getPeer() - if isinstance(remote, address.IPv4Address): - return "%s:%d" % (remote.host, remote.port) - # loopback is a non-IPv4Address - return str(remote) diff --git a/src/allmydata/util/sibpath.py b/src/allmydata/util/sibpath.py deleted file mode 100644 index 80a2801a3..000000000 --- a/src/allmydata/util/sibpath.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import sys -from twisted.python.util import sibpath as tsibpath - -def sibpath(path, sibling): - """ - Looks for a named sibling relative to the given path. If such a file - exists, its path will be returned, otherwise a second search will be - made for the named sibling relative to the path of the executable - currently running. This is useful in the case that something built - with py2exe, for example, needs to find data files relative to its - install. Note hence that care should be taken not to search for - private package files whose names might collide with files which might - be found installed alongside the python interpreter itself. If no - file is found in either place, the sibling relative to the given path - is returned, likely leading to a file not found error. - """ - sib = tsibpath(path, sibling) - if not os.path.exists(sib): - exe_sib = tsibpath(sys.executable, sibling) - if os.path.exists(exe_sib): - return exe_sib - return sib - diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py index c4c63f61a..4ca19c01c 100644 --- a/src/allmydata/util/tor_provider.py +++ b/src/allmydata/util/tor_provider.py @@ -211,6 +211,8 @@ def create_config(reactor, cli_config): "tor_onion.privkey") privkeyfile = os.path.join(private_dir, "tor_onion.privkey") with open(privkeyfile, "wb") as f: + if isinstance(privkey, str): + privkey = privkey.encode("ascii") f.write(privkey) # tahoe_config_tor: this is a dictionary of keys/values to add to the diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py deleted file mode 100644 index 2dfc24a1b..000000000 --- a/src/allmydata/util/verlib.py +++ /dev/null @@ -1,336 +0,0 @@ -""" -"Rational" version definition and parsing for DistutilsVersionFight -discussion at PyCon 2009. - -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import re - - -class IrrationalVersionError(Exception): - """This is an irrational version.""" - pass - -class HugeMajorVersionNumError(IrrationalVersionError): - """An irrational version because the major version number is huge - (often because a year or date was used). - - See `error_on_huge_major_num` option in `NormalizedVersion` for details. - This guard can be disabled by setting that option False. - """ - pass - -# A marker used in the second and third parts of the `parts` tuple, for -# versions that don't have those segments, to sort properly. An example -# of versions in sort order ('highest' last): -# 1.0b1 ((1,0), ('b',1), ('f',)) -# 1.0.dev345 ((1,0), ('f',), ('dev', 345)) -# 1.0 ((1,0), ('f',), ('f',)) -# 1.0.post256.dev345 ((1,0), ('f',), ('f', 'post', 256, 'dev', 345)) -# 1.0.post345 ((1,0), ('f',), ('f', 'post', 345, 'f')) -# ^ ^ ^ -# 'b' < 'f' ---------------------/ | | -# | | -# 'dev' < 'f' < 'post' -------------------/ | -# | -# 'dev' < 'f' ----------------------------------------------/ -# Other letters would do, but 'f' for 'final' is kind of nice. -FINAL_MARKER = ('f',) - -VERSION_RE = re.compile(r''' - ^ - (?P\d+\.\d+) # minimum 'N.N' - (?P(?:\.\d+)*) # any number of extra '.N' segments - (?: - (?P[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate - # 'rc'= alias for release candidate - (?P\d+(?:\.\d+)*) - )? - (?P(\.post(?P\d+))?(\.dev(?P\d+))?)? - $''', re.VERBOSE) - -class NormalizedVersion(object): - """A rational version. - - Good: - 1.2 # equivalent to "1.2.0" - 1.2.0 - 1.2a1 - 1.2.3a2 - 1.2.3b1 - 1.2.3c1 - 1.2.3.4 - TODO: fill this out - - Bad: - 1 # mininum two numbers - 1.2a # release level must have a release serial - 1.2.3b - """ - def __init__(self, s, error_on_huge_major_num=True): - """Create a NormalizedVersion instance from a version string. - - @param s {str} The version string. - @param error_on_huge_major_num {bool} Whether to consider an - apparent use of a year or full date as the major version number - an error. Default True. One of the observed patterns on PyPI before - the introduction of `NormalizedVersion` was version numbers like this: - 2009.01.03 - 20040603 - 2005.01 - This guard is here to strongly encourage the package author to - use an alternate version, because a release deployed into PyPI - and, e.g. downstream Linux package managers, will forever remove - the possibility of using a version number like "1.0" (i.e. - where the major number is less than that huge major number). - """ - self._parse(s, error_on_huge_major_num) - - @classmethod - def from_parts(cls, version, prerelease=FINAL_MARKER, - devpost=FINAL_MARKER): - return cls(cls.parts_to_str((version, prerelease, devpost))) - - def _parse(self, s, error_on_huge_major_num=True): - """Parses a string version into parts.""" - match = VERSION_RE.search(s) - if not match: - raise IrrationalVersionError(s) - - groups = match.groupdict() - parts = [] - - # main version - block = self._parse_numdots(groups['version'], s, False, 2) - extraversion = groups.get('extraversion') - if extraversion not in ('', None): - block += self._parse_numdots(extraversion[1:], s) - parts.append(tuple(block)) - - # prerelease - prerel = groups.get('prerel') - if prerel is not None: - block = [prerel] - block += self._parse_numdots(groups.get('prerelversion'), s, - pad_zeros_length=1) - parts.append(tuple(block)) - else: - parts.append(FINAL_MARKER) - - # postdev - if groups.get('postdev'): - post = groups.get('post') - dev = groups.get('dev') - postdev = [] - if post is not None: - postdev.extend([FINAL_MARKER[0], 'post', int(post)]) - if dev is None: - postdev.append(FINAL_MARKER[0]) - if dev is not None: - postdev.extend(['dev', int(dev)]) - parts.append(tuple(postdev)) - else: - parts.append(FINAL_MARKER) - self.parts = tuple(parts) - if error_on_huge_major_num and self.parts[0][0] > 1980: - raise HugeMajorVersionNumError("huge major version number, %r, " - "which might cause future problems: %r" % (self.parts[0][0], s)) - - def _parse_numdots(self, s, full_ver_str, drop_trailing_zeros=True, - pad_zeros_length=0): - """Parse 'N.N.N' sequences, return a list of ints. - - @param s {str} 'N.N.N...' sequence to be parsed - @param full_ver_str {str} The full version string from which this - comes. Used for error strings. - @param drop_trailing_zeros {bool} Whether to drop trailing zeros - from the returned list. Default True. - @param pad_zeros_length {int} The length to which to pad the - returned list with zeros, if necessary. Default 0. - """ - nums = [] - for n in s.split("."): - if len(n) > 1 and n[0] == '0': - raise IrrationalVersionError("cannot have leading zero in " - "version number segment: '%s' in %r" % (n, full_ver_str)) - nums.append(int(n)) - if drop_trailing_zeros: - while nums and nums[-1] == 0: - nums.pop() - while len(nums) < pad_zeros_length: - nums.append(0) - return nums - - def __str__(self): - return self.parts_to_str(self.parts) - - @classmethod - def parts_to_str(cls, parts): - """Transforms a version expressed in tuple into its string - representation.""" - # XXX This doesn't check for invalid tuples - main, prerel, postdev = parts - s = '.'.join(str(v) for v in main) - if prerel is not FINAL_MARKER: - s += prerel[0] - s += '.'.join(str(v) for v in prerel[1:]) - if postdev and postdev is not FINAL_MARKER: - if postdev[0] == 'f': - postdev = postdev[1:] - i = 0 - while i < len(postdev): - if i % 2 == 0: - s += '.' - s += str(postdev[i]) - i += 1 - return s - - def __repr__(self): - return "%s('%s')" % (self.__class__.__name__, self) - - def _cannot_compare(self, other): - raise TypeError("cannot compare %s and %s" - % (type(self).__name__, type(other).__name__)) - - def __eq__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts == other.parts - - def __lt__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts < other.parts - - def __ne__(self, other): - return not self.__eq__(other) - - def __gt__(self, other): - return not (self.__lt__(other) or self.__eq__(other)) - - def __le__(self, other): - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) - -def suggest_normalized_version(s): - """Suggest a normalized version close to the given version string. - - If you have a version string that isn't rational (i.e. NormalizedVersion - doesn't like it) then you might be able to get an equivalent (or close) - rational version from this function. - - This does a number of simple normalizations to the given string, based - on observation of versions currently in use on PyPI. Given a dump of - those version during PyCon 2009, 4287 of them: - - 2312 (53.93%) match NormalizedVersion without change - - with the automatic suggestion - - 3474 (81.04%) match when using this suggestion method - - @param s {str} An irrational version string. - @returns A rational version string, or None, if couldn't determine one. - """ - try: - NormalizedVersion(s) - return s # already rational - except IrrationalVersionError: - pass - - rs = s.lower() - - # part of this could use maketrans - for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), - ('beta', 'b'), ('rc', 'c'), ('-final', ''), - ('-pre', 'c'), - ('-release', ''), ('.release', ''), ('-stable', ''), - ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), - ('final', '')): - rs = rs.replace(orig, repl) - - # if something ends with dev or pre, we add a 0 - rs = re.sub(r"pre$", r"pre0", rs) - rs = re.sub(r"dev$", r"dev0", rs) - - # if we have something like "b-2" or "a.2" at the end of the - # version, that is pobably beta, alpha, etc - # let's remove the dash or dot - rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs) - - # 1.0-dev-r371 -> 1.0.dev371 - # 0.1-dev-r79 -> 0.1.dev79 - rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) - - # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 - rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) - - # Clean: v0.3, v1.0 - if rs.startswith('v'): - rs = rs[1:] - - # Clean leading '0's on numbers. - #TODO: unintended side-effect on, e.g., "2003.05.09" - # PyPI stats: 77 (~2%) better - rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) - - # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers - # zero. - # PyPI stats: 245 (7.56%) better - rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) - - # the 'dev-rNNN' tag is a dev tag - rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) - - # clean the - when used as a pre delimiter - rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) - - # a terminal "dev" or "devel" can be changed into ".dev0" - rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) - - # a terminal "dev" can be changed into ".dev0" - rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) - - # a terminal "final" or "stable" can be removed - rs = re.sub(r"(final|stable)$", "", rs) - - # The 'r' and the '-' tags are post release tags - # 0.4a1.r10 -> 0.4a1.post10 - # 0.9.33-17222 -> 0.9.33.post17222 - # 0.9.33-r17222 -> 0.9.33.post17222 - rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) - - # Clean 'r' instead of 'dev' usage: - # 0.9.33+r17222 -> 0.9.33.dev17222 - # 1.0dev123 -> 1.0.dev123 - # 1.0.git123 -> 1.0.dev123 - # 1.0.bzr123 -> 1.0.dev123 - # 0.1a0dev.123 -> 0.1a0.dev123 - # PyPI stats: ~150 (~4%) better - rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) - - # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: - # 0.2.pre1 -> 0.2c1 - # 0.2-c1 -> 0.2c1 - # 1.0preview123 -> 1.0c123 - # PyPI stats: ~21 (0.62%) better - rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) - - - # Tcl/Tk uses "px" for their post release markers - rs = re.sub(r"p(\d+)$", r".post\1", rs) - - try: - NormalizedVersion(rs) - return rs # already rational - except IrrationalVersionError: - pass - return None diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index 40c38fa30..fd9fc73e2 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,11 +1,39 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import yaml -# Announcements contain unicode, because they come from JSON. We tell PyYAML -# to give us unicode instead of str/bytes. -def construct_unicode(loader, node): - return node.value -yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", - construct_unicode) + +if PY2: + # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. + # + # >>> yaml.safe_load(yaml.safe_dump(u"hello")) + # 'hello' + # >>> yaml.safe_load(yaml.safe_dump(u"hello\u1234")) + # u'hello\u1234' + # + # In other words, Unicode strings get roundtripped to byte strings, but + # only sometimes. + # + # In order to ensure unicode stays unicode, we add a configuration saying + # that the YAML String Language-Independent Type ("a sequence of zero or + # more Unicode characters") should be the underlying Unicode string object, + # rather than converting to bytes when possible. + # + # Reference: https://yaml.org/type/str.html + def construct_unicode(loader, node): + return node.value + yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", + construct_unicode) def safe_load(f): return yaml.safe_load(f) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index bb84a2e70..bf89044a3 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -90,6 +90,7 @@ from allmydata.util.time_format import ( ) from allmydata.util.encodingutil import ( quote_output, + quote_output_u, to_bytes, ) from allmydata.util import abbreviate @@ -324,7 +325,7 @@ def humanize_exception(exc): return ("There was already a child by that name, and you asked me " "to not replace it.", http.CONFLICT) if isinstance(exc, NoSuchChildError): - quoted_name = quote_output(exc.args[0], encoding="utf-8", quotemarks=False) + quoted_name = quote_output_u(exc.args[0], quotemarks=False) return ("No such child: %s" % quoted_name, http.NOT_FOUND) if isinstance(exc, NotEnoughSharesError): t = ("NotEnoughSharesError: This indicates that some " diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index a78e9cd12..a79440eb9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -8,8 +8,6 @@ from __future__ import ( division, ) -import json - from autobahn.twisted.resource import WebSocketResource from autobahn.twisted.websocket import ( WebSocketServerFactory, @@ -21,6 +19,8 @@ from twisted.web.resource import ( Resource, ) +from allmydata.util import jsonbytes as json + class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ @@ -47,10 +47,7 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - encoded = json.dumps(message) - if isinstance(encoded, str): - # On Python 3 dumps() returns Unicode... - encoded = encoded.encode("utf-8") + encoded = json.dumps_bytes(message, any_bytes=True) self.sendMessage(encoded) def onOpen(self): diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b7dc8b5f4..1debc1d10 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -318,7 +318,7 @@ class Root(MultiFormatResource): } version = server.get_version() if version is not None: - description[u"version"] = version["application-version"] + description[u"version"] = version[b"application-version"] return description diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 0401fb586..158d897f9 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1173,7 +1173,8 @@ class MapupdateStatusElement(Element): def privkey_from(self, req, tag): server = self._update_status.get_privkey_from() if server: - return tag(tags.li("Got privkey from: [%s]" % server.get_name())) + return tag(tags.li("Got privkey from: [%s]" % str( + server.get_name(), "utf-8"))) else: return tag diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 3b5437a7b..e8d05a659 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,5 +1,8 @@ from __future__ import print_function +from future.utils import PY3 +from past.builtins import unicode + # This code isn't loadable or sensible except on Windows. Importers all know # this and are careful. Normally I would just let an import error from ctypes # explain any mistakes but Mypy also needs some help here. This assert @@ -121,6 +124,10 @@ def initialize(): SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) + if PY3: + # The rest of this appears to be Python 2-specific + return + original_stderr = sys.stderr # If any exception occurs in this code, we'll probably try to print it on stderr, diff --git a/src/allmydata/windows/registry.py b/src/allmydata/windows/registry.py index c1f695d2b..a5426cb75 100644 --- a/src/allmydata/windows/registry.py +++ b/src/allmydata/windows/registry.py @@ -1,5 +1,5 @@ import sys -import _winreg +import winreg _AMD_KEY = r"Software\Allmydata" _BDIR_KEY = 'Base Dir Path' @@ -22,19 +22,19 @@ def get_registry_setting(key, name, _topkey=None): @param name: The name of the setting we are querying. @type name: String """ - topkeys = [_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE] + topkeys = [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE] if _topkey: topkeys.insert(0, _topkey) for topkey in topkeys: try: - regkey = _winreg.OpenKey(topkey, key) + regkey = winreg.OpenKey(topkey, key) - sublen, vallen, timestamp = _winreg.QueryInfoKey(regkey) - for validx in xrange(vallen): - keyname, value, keytype = _winreg.EnumValue(regkey, validx) - if keyname == name and keytype == _winreg.REG_SZ: + sublen, vallen, timestamp = winreg.QueryInfoKey(regkey) + for validx in range(vallen): + keyname, value, keytype = winreg.EnumValue(regkey, validx) + if keyname == name and keytype == winreg.REG_SZ: return value except WindowsError: @@ -42,27 +42,27 @@ def get_registry_setting(key, name, _topkey=None): # We didn't find the key: raise KeyError(key, name, "registry setting not found") -def set_registry_setting(key, name, data, reg_type=_winreg.REG_SZ, - _topkey=_winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): +def set_registry_setting(key, name, data, reg_type=winreg.REG_SZ, + _topkey=winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): """ Sets a registry setting. defaults to string values (REG_SZ) - overridable with reg_type. """ try: - regkey = _winreg.OpenKey(_topkey, key, 0, _winreg.KEY_SET_VALUE) + regkey = winreg.OpenKey(_topkey, key, 0, winreg.KEY_SET_VALUE) except WindowsError: if create_key_if_missing: - regkey = _winreg.CreateKey(_topkey, key) + regkey = winreg.CreateKey(_topkey, key) else: raise KeyError(key, "registry key not found") try: - _winreg.DeleteValue(regkey, name) + winreg.DeleteValue(regkey, name) except: pass - _winreg.SetValueEx(regkey, name, 0, reg_type, data) + winreg.SetValueEx(regkey, name, 0, reg_type, data) def get_registry_value(keyname): """ diff --git a/towncrier.pyproject.toml b/towncrier.toml similarity index 91% rename from towncrier.pyproject.toml rename to towncrier.toml index 26627c3f9..b8b561a98 100644 --- a/towncrier.pyproject.toml +++ b/towncrier.toml @@ -37,6 +37,11 @@ name = "Configuration Changes" showcontent = true + [[tool.towncrier.type]] + directory = "documentation" + name = "Documentation Changes" + showcontent = true + [[tool.towncrier.type]] directory = "removed" name = "Removed Features" diff --git a/tox.ini b/tox.ini index 8908142f4..9b0f71038 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,22 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# Map Python versions in GitHub Actions to tox environments to run, for use by +# the tox-gh-actions package. +[gh-actions] +python = + 2.7: py27-coverage,codechecks + 3.6: py36-coverage + 3.7: py37-coverage,typechecks,codechecks3 + 3.8: py38-coverage + 3.9: py39-coverage + pypy-3.7: pypy3 + [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -25,9 +36,11 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - pip==19.1.1 - setuptools==41.0.1 - wheel==0.33.4 + # + # For now these are versions that support Python 2. + pip==20.3.4 + setuptools==44.1.1 + wheel==0.36.2 subunitreporter==19.3.2 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is @@ -50,8 +63,7 @@ extras = test setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - !py36: TEST_SUITE = allmydata - py36: TEST_SUITE = allmydata.test.python3_tests + TEST_SUITE = allmydata commands = # As an aid to debugging, dump all of the Python packages and their @@ -85,20 +97,40 @@ commands = coverage report +[testenv:integration3] +basepython = python3 +setenv = + COVERAGE_PROCESS_START=.coveragerc +commands = + python --version + # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' + py.test --timeout=1800 --coverage -v {posargs:integration} + coverage combine + coverage report + + +# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 +# since that can give different results than flake8 on Python 3. [testenv:codechecks] basepython = python2.7 +setenv = + # If no positional arguments are given, try to run the checks on the + # entire codebase, including various pieces of supporting code. + DEFAULT_FILES=src integration static misc setup.py +commands = + flake8 {posargs:{env:DEFAULT_FILES}} + + +[testenv:codechecks3] +basepython = python3 +deps = + # Newer versions of PyLint have buggy configuration + # (https://github.com/PyCQA/pylint/issues/4574), so stick to old version + # for now. + pylint < 2.5 # On macOS, git inside of towncrier needs $HOME. passenv = HOME -whitelist_externals = - /bin/mv setenv = - # Workaround an error when towncrier is run under the VCS hook, - # https://stackoverflow.com/a/4027726/624787: - # File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/codechecks/lib/python2.7/site-packages/towncrier/check.py", line 44, in __main - # .decode(getattr(sys.stdout, "encoding", "utf8")) - # `TypeError: decode() argument 1 must be string, not None` - PYTHONIOENCODING=utf_8 - # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. DEFAULT_FILES=src integration static misc setup.py @@ -107,13 +139,15 @@ commands = python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}} python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}} - python misc/coding_tools/check-miscaptures.py {posargs:{env:DEFAULT_FILES}} + # PyLint has other useful checks, might want to enable them: + # http://pylint.pycqa.org/en/latest/technical_reference/features.html + pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}} # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --pyproject towncrier.pyproject.toml + # file. See towncrier.toml for legal values. + python -m towncrier.check --config towncrier.toml [testenv:typechecks] @@ -121,7 +155,11 @@ basepython = python3 skip_install = True deps = mypy - git+https://github.com/Shoobx/mypy-zope + mypy-zope + types-mock + types-six + types-PyYAML + types-pkg_resources git+https://github.com/warner/foolscap # Twisted 21.2.0 introduces some type hints which we are not yet # compatible with. @@ -132,62 +170,26 @@ commands = mypy src [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# see comment in [testenv] about "certifi" -whitelist_externals = mv deps = - certifi - towncrier >= 19.2 + # see comment in [testenv] about "certifi" + certifi + towncrier==21.3.0 commands = - # With pip >= 10 the existence of pyproject.toml (which we are - # required to have to configure towncrier) triggers a "build - # isolation" mode which prevents anything from working. Avoid - # triggering that pip behavior by keeping the towncrier configuration - # somewhere else and only bringing it in when it's actually needed - # (after pip is done). - # - # Some discussion is available at - # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. - mv towncrier.pyproject.toml pyproject.toml - - # towncrier 19.2 + works with python2.7 - python -m towncrier --draft - - # put it back - mv pyproject.toml towncrier.pyproject.toml + python -m towncrier --draft --config towncrier.toml [testenv:news] -passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# see comment in [testenv] about "certifi" -whitelist_externals = mv +# On macOS, git invoked from Tox needs $HOME. +passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HOME +whitelist_externals = + git deps = - certifi - towncrier >= 19.2 + # see comment in [testenv] about "certifi" + certifi + towncrier==21.3.0 commands = - # With pip >= 10 the existence of pyproject.toml (which we are - # required to have to configure towncrier) triggers a "build - # isolation" mode which prevents anything from working. Avoid - # triggering that pip behavior by keeping the towncrier configuration - # somewhere else and only bringing it in when it's actually needed - # (after pip is done). - # - # Some discussion is available at - # https://github.com/pypa/pip/issues/5696 - # - # towncrier post 19.2 (unreleased as of this writing) adds a --config - # option that can be used instead of this file shuffling. - mv towncrier.pyproject.toml pyproject.toml - - # towncrier 19.2 + works with python2.7 - python -m towncrier --yes - - # put it back - mv pyproject.toml towncrier.pyproject.toml - - # commit the changes - git commit -m "update NEWS.txt for release" + python -m towncrier --yes --config towncrier.toml + # commit the changes + git commit -m "update NEWS.txt for release" [testenv:deprecations] commands = @@ -234,7 +236,7 @@ deps = # normal install is not needed for docs, and slows things down skip_install = True commands = - sphinx-build -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html + sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] # We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) @@ -247,6 +249,7 @@ deps = # PyInstaller 4.0 drops Python 2 support. When we finish porting to # Python 3 we can reconsider this constraint. pyinstaller < 4.0 + pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1