Merge remote-tracking branch 'origin/master' into 3528.test_cli-no-mock

2024-12-19 21:17:54 +00:00 · 2021-02-12 13:35:03 -05:00 · 2021-02-12 13:35:03 -05:00 · 91490fc2d5
commit 91490fc2d5
parent 916ddd590e 33d566ec87
181 changed files with 4361 additions and 3058 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -29,7 +29,7 @@ workflows:
      - "debian-9": &DOCKERHUB_CONTEXT
          context: "dockerhub-auth"

-      - "debian-8":
+      - "debian-10":
          <<: *DOCKERHUB_CONTEXT
          requires:
            - "debian-9"
@ -86,9 +86,7 @@ workflows:
            # integration tests.
            - "debian-9"

-      # Generate the underlying data for a visualization to aid with Python 3
-      # porting.
-      - "build-porting-depgraph":
+      - "typechecks":
          <<: *DOCKERHUB_CONTEXT

  images:
@ -104,7 +102,7 @@ workflows:
                - "master"

    jobs:
-      - "build-image-debian-8":
+      - "build-image-debian-10":
          <<: *DOCKERHUB_CONTEXT
      - "build-image-debian-9":
          <<: *DOCKERHUB_CONTEXT
@ -210,7 +208,7 @@ jobs:
      # filenames and argv).
      LANG: "en_US.UTF-8"
      # Select a tox environment to run for this job.
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py27"
      # Additional arguments to pass to tox.
      TAHOE_LAFS_TOX_ARGS: ""
      # The path in which test artifacts will be placed.
@ -220,7 +218,7 @@ jobs:
      WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
      PIP_FIND_LINKS: "file:///tmp/wheelhouse"
      # Upload the coverage report.
-      UPLOAD_COVERAGE: "yes"
+      UPLOAD_COVERAGE: ""

    # pip cannot install packages if the working directory is not readable.
    # We want to run a lot of steps as nobody instead of as root.
@ -274,11 +272,11 @@ jobs:
            fi


-  debian-8:
+  debian-10:
    <<: *DEBIAN
    docker:
      - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/debian:8-py2.7"
+        image: "tahoelafsci/debian:10-py2.7"
        user: "nobody"


@ -373,7 +371,7 @@ jobs:
      # this reporter on Python 3.  So drop that and just specify the
      # reporter.
      TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"


  ubuntu-20-04:
@ -448,32 +446,17 @@ jobs:
            # them in parallel.
            nix-build --cores 3 --max-jobs 2 nix/

-  # Generate up-to-date data for the dependency graph visualizer.
-  build-porting-depgraph:
-    # Get a system in which we can easily install Tahoe-LAFS and all its
-    # dependencies.  The dependency graph analyzer works by executing the code.
-    # It's Python, what do you expect?
-    <<: *DEBIAN
+  typechecks:
+    docker:
+      - <<: *DOCKERHUB_AUTH
+        image: "tahoelafsci/ubuntu:18.04-py3"

    steps:
      - "checkout"
-
-      - add_ssh_keys:
-          fingerprints:
-            # Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
-            # This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
-            - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
-
      - run:
-          name: "Setup Python Environment"
+          name: "Validate Types"
          command: |
-            /tmp/venv/bin/pip install -e /tmp/project
-
-      - run:
-          name: "Generate dependency graph data"
-          command: |
-            . /tmp/venv/bin/activate
-            ./misc/python3/depgraph.sh
+            /tmp/venv/bin/tox -e typechecks

  build-image: &BUILD_IMAGE
    # This is a template for a job to build a Docker image that has as much of
@ -514,12 +497,12 @@ jobs:
            docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}


-  build-image-debian-8:
+  build-image-debian-10:
    <<: *BUILD_IMAGE

    environment:
      DISTRO: "debian"
-      TAG: "8"
+      TAG: "10"
      PYTHON_VERSION: "2.7"


--- a/.codecov.yml
+++ b/.codecov.yml
@ -1,42 +0,0 @@
-# Override defaults for codecov.io checks.
-#
-# Documentation is at https://docs.codecov.io/docs/codecov-yaml;
-# reference is at https://docs.codecov.io/docs/codecovyml-reference.
-#
-# To validate this file, use:
-#
-#   curl --data-binary @.codecov.yml https://codecov.io/validate
-#
-# Codecov's defaults seem to leave red marks in GitHub CI checks in a
-# rather arbitrary manner, probably because of non-determinism in
-# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891)
-# and maybe because computers are bad with floating point numbers.
-
-# Allow coverage percentage a precision of zero decimals, and round to
-# the nearest number (for example, 89.957 to to 90; 89.497 to 89%).
-# Coverage above 90% is good, below 80% is bad.
-coverage:
-  round: nearest
-  range: 80..90
-  precision: 0
-
-  # Aim for a target test coverage of 90% in codecov/project check (do
-  # not allow project coverage to drop below that), and allow
-  # codecov/patch a threshold of 1% (allow coverage in changes to drop
-  # by that much, and no less).  That should be good enough for us.
-  status:
-    project:
-      default:
-        target: 90%
-        threshold: 1%
-    patch:
-      default:
-        threshold: 1%
-
-
-codecov:
-  # This is a public repository so supposedly we don't "need" to use an upload
-  # token.  However, using one makes sure that CI jobs running against forked
-  # repositories have coverage uploaded to the right place in codecov so
-  # their reports aren't incomplete.
-  token: "abf679b6-e2e6-4b33-b7b5-6cfbd41ee691"
--- a/.github/CONTRIBUTING.rst
+++ b/.github/CONTRIBUTING.rst
@ -17,4 +17,4 @@ Examples of contributions include:
 * `Patch reviews <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/PatchReviewProcess>`_

 Before authoring or reviewing a patch,
-please familiarize yourself with the `coding standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`_.
+please familiarize yourself with the `Coding Standards <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_.
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -30,17 +30,37 @@ jobs:
        with:
          args: install vcpython27

+      # See https://github.com/actions/checkout. A fetch-depth of 0
+      # fetches all tags and branches.
      - name: Check out Tahoe-LAFS sources
        uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}

+      # To use pip caching with GitHub Actions in an OS-independent
+      # manner, we need `pip cache dir` command, which became
+      # available since pip v20.1+.  At the time of writing this,
+      # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and
+      # windows-latest, and pip v20.3.1 for macos-latest.
+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      # See https://github.com/actions/cache
+      - name: Use pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
      - name: Install Python packages
        run: |
          pip install --upgrade codecov tox setuptools
@ -59,11 +79,110 @@ jobs:
          name: eliot.log
          path: eliot.log

-      - name: Upload coverage report
-        uses: codecov/codecov-action@v1
-        with:
-          token: abf679b6-e2e6-4b33-b7b5-6cfbd41ee691
-          file: coverage.xml
+      # Upload this job's coverage data to Coveralls.  While there is a GitHub
+      # Action for this, as of Jan 2021 it does not support Python coverage
+      # files - only lcov files.  Therefore, we use coveralls-python, the
+      # coveralls.io-supplied Python reporter, for this.
+      - name: "Report Coverage to Coveralls"
+        run: |
+          pip install coveralls
+          python -m coveralls
+        env:
+          # Some magic value required for some magic reason.
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          # Help coveralls identify our project.
+          COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
+          # Every source of coverage reports needs a unique "flag name".
+          # Construct one by smashing a few variables from the matrix together
+          # here.
+          COVERALLS_FLAG_NAME: "run-${{ matrix.os }}-${{ matrix.python-version }}"
+          # Mark the data as just one piece of many because we have more than
+          # one instance of this job (Windows, macOS) which collects and
+          # reports coverage.  This is necessary to cause Coveralls to merge
+          # multiple coverage results into a single report.  Note the merge
+          # only happens when we "finish" a particular build, as identified by
+          # its "build_num" (aka "service_number").
+          COVERALLS_PARALLEL: true
+
+  # Tell Coveralls that we're done reporting coverage data.  Since we're using
+  # the "parallel" mode where more than one coverage data file is merged into
+  # a single report, we have to tell Coveralls when we've uploaded all of the
+  # data files.  This does it.  We make sure it runs last by making it depend
+  # on *all* of the coverage-collecting jobs.
+  finish-coverage-report:
+    # There happens to just be one coverage-collecting job at the moment.  If
+    # the coverage reports are broken and someone added more
+    # coverage-collecting jobs to this workflow but didn't update this, that's
+    # why.
+    needs:
+      - "coverage"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "Check out Tahoe-LAFS sources"
+        uses: "actions/checkout@v2"
+
+      - name: "Finish Coveralls Reporting"
+        run: |
+          # coveralls-python does have a `--finish` option but it doesn't seem
+          # to work, at least for us.
+          # https://github.com/coveralls-clients/coveralls-python/issues/248
+          #
+          # But all it does is this simple POST so we can just send it
+          # ourselves.  The only hard part is guessing what the POST
+          # parameters mean.  And I've done that for you already.
+          #
+          # Since the build is done I'm going to guess that "done" is a fine
+          # value for status.
+          #
+          # That leaves "build_num".  The coveralls documentation gives some
+          # hints about it.  It suggests using $CIRCLE_WORKFLOW_ID if your job
+          # is on CircleCI.  CircleCI documentation says this about
+          # CIRCLE_WORKFLOW_ID:
+          #
+          # Observation of the coveralls.io web interface, logs from the
+          # coveralls command in action, and experimentation suggests the
+          # value for PRs is something more like:
+          #
+          #    <GIT MERGE COMMIT HASH>-PR-<PR NUM>
+          #
+          # For branches, it's just the git branch tip hash.
+
+          # For pull requests, refs/pull/<PR NUM>/merge was just checked out
+          # by so HEAD will refer to the right revision.  For branches, HEAD
+          # is also the tip of the branch.
+          REV=$(git rev-parse HEAD)
+
+          # We can get the PR number from the "context".
+          #
+          # https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request
+          #
+          # (via <https://github.community/t/github-ref-is-inconsistent/17728/3>).
+          #
+          # If this is a pull request, `github.event` is a `pull_request`
+          # structure which has `number` right in it.
+          #
+          # If this is a push, `github.event` is a `push` instead but we only
+          # need the revision to construct the build_num.
+
+          PR=${{ github.event.number }}
+
+          if [ "${PR}" = "" ]; then
+              BUILD_NUM=$REV
+          else
+              BUILD_NUM=$REV-PR-$PR
+          fi
+          REPO_NAME=$GITHUB_REPOSITORY
+
+          curl \
+            -k \
+            https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \
+            -d \
+            "payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME"
+        env:
+          # Some magic value required for some magic reason.
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          # Help coveralls identify our project.
+          COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"

  integration:
    runs-on: ${{ matrix.os }}
@ -103,15 +222,27 @@ jobs:

      - name: Check out Tahoe-LAFS sources
        uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}

+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Use pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
      - name: Install Python packages
        run: |
          pip install --upgrade tox
@ -155,15 +286,27 @@ jobs:

      - name: Check out Tahoe-LAFS sources
        uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}

+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Use pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
      - name: Install Python packages
        run: |
          pip install --upgrade tox
--- a/7
+++ b/7
@ -206,4 +206,9 @@ D: various bug-fixes and features

 N: Viktoriia Savchuk
 W: https://twitter.com/viktoriiasvchk
-D: Developer community focused improvements on the README file.
+D: Developer community focused improvements on the README file.
+
+N: Lukas Pirl
+E: tahoe@lukas-pirl.de
+W: http://lukas-pirl.de
+D: Buildslaves (Debian, Fedora, CentOS; 2016-2021)
--- a/README.rst
+++ b/README.rst
@ -6,7 +6,7 @@ Free and Open decentralized data store

 `Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.

-|Contributor Covenant|  |readthedocs|  |travis|  |circleci|  |codecov|
+|Contributor Covenant|  |readthedocs|  |travis|  |circleci|  |coveralls|


 Table of contents
@ -125,9 +125,9 @@ See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL ex
 .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
    :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs

-.. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
-    :alt: test coverage percentage
-    :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master
+.. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg
+    :alt: code coverage
+    :target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs

 .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
    :alt: code of conduct
--- a/docs/INSTALL.rst
+++ b/docs/INSTALL.rst
@ -173,7 +173,9 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run
 Install From a Source Tarball
 -----------------------------

-You can also install directly from the source tarball URL::
+You can also install directly from the source tarball URL. To verify
+signatures, first see verifying_signatures_ and replace the URL in the
+following instructions with the local filename.

 % virtualenv venv
 New python executable in ~/venv/bin/python2.7
@ -189,6 +191,40 @@ You can also install directly from the source tarball URL::
 tahoe-lafs: 1.14.0
 ...

+.. _verifying_signatures:
+
+Verifying Signatures
+--------------------
+
+First download the source tarball and then any signatures. There are several
+developers who are able to produce signatures for a release. A release may
+have multiple signatures. All should be valid and you should confirm at least
+one of them (ideally, confirm all).
+
+This statement, signed by the existing Tahoe release-signing key, attests to
+those developers authorized to sign a Tahoe release:
+
+.. include:: developer-release-signatures
+   :code:
+
+Signatures are made available beside the release. So for example, a release
+like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might
+have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and
+``tahoe-lafs-1.16.0.tar.bz2.warner.asc``.
+
+To verify the signatures using GnuPG::
+
+  % gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2
+  gpg: Signature made XXX
+  gpg:                using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7
+  gpg: Good signature from "meejah <meejah@meejah.ca>" [full]
+  % gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2
+  gpg: Signature made XXX
+  gpg:                using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF
+  gpg: Good signature from "Brian Warner <warner@lothar.com>" [full]
+
+
+
 Extras
 ------

--- a/docs/README.txt
+++ b/docs/README.txt
--- a/docs/about.rst
+++ b/docs/about.rst
@ -67,12 +67,12 @@ Here's how it works:
 A "storage grid" is made up of a number of storage servers.  A storage server
 has direct attached storage (typically one or more hard disks).  A "gateway"
 communicates with storage nodes, and uses them to provide access to the
-grid over protocols such as HTTP(S), SFTP or FTP.
+grid over protocols such as HTTP(S) and SFTP.

 Note that you can find "client" used to refer to gateway nodes (which act as
 a client to storage servers), and also to processes or programs connecting to
 a gateway node and performing operations on the grid -- for example, a CLI
-command, Web browser, SFTP client, or FTP client.
+command, Web browser, or SFTP client.

 Users do not rely on storage servers to provide *confidentiality* nor
 *integrity* for their data -- instead all of the data is encrypted and
--- a/docs/conf.py
+++ b/docs/conf.py
@ -28,7 +28,7 @@ import os
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = []
+extensions = ['recommonmark']

 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@ -36,7 +36,7 @@ templates_path = ['_templates']
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ['.rst', '.md']

 # The encoding of source files.
 #source_encoding = 'utf-8-sig'
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@ -81,7 +81,6 @@ Client/server nodes provide one or more of the following services:

 * web-API service
 * SFTP service
-* FTP service
 * helper service
 * storage service.

@ -708,12 +707,12 @@ CLI
    file store, uploading/downloading files, and creating/running Tahoe
    nodes. See :doc:`frontends/CLI` for details.

-SFTP, FTP
+SFTP

-    Tahoe can also run both SFTP and FTP servers, and map a username/password
+    Tahoe can also run SFTP servers, and map a username/password
    pair to a top-level Tahoe directory. See :doc:`frontends/FTP-and-SFTP`
-    for instructions on configuring these services, and the ``[sftpd]`` and
-    ``[ftpd]`` sections of ``tahoe.cfg``.
+    for instructions on configuring this service, and the ``[sftpd]``
+    section of ``tahoe.cfg``.


 Storage Server Configuration
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@ -0,0 +1 @@
+.. include:: ../.github/CONTRIBUTING.rst
--- a/docs/developer-release-signatures
+++ b/docs/developer-release-signatures
@ -0,0 +1,42 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA512
+
+
+January 20, 2021
+
+Any of the following core Tahoe contributers may sign a release. Each
+release MUST be signed by at least one developer but MAY have
+additional signatures. Each developer independently produces a
+signature which is made available beside Tahoe releases after 1.15.0
+
+This statement is signed by the existing Tahoe release key. Any future
+such statements may be signed by it OR by any two developers (for
+example, to add or remove developers from the list).
+
+meejah
+0xC2602803128069A7
+9D5A 2BD5 688E CB88 9DEB CD3F C260 2803 1280 69A7
+https://meejah.ca/meejah.asc
+
+jean-paul calderone (exarkun)
+0xE27B085EDEAA4B1B
+96B9 C5DA B2EA 9EB6 7941 9DB7 E27B 085E DEAA 4B1B
+https://twistedmatrix.com/~exarkun/E27B085EDEAA4B1B.asc
+
+brian warner (lothar)
+0x863333C265497810
+5810 F125 7F8C F753 7753  895A 8633 33C2 6549 7810
+https://www.lothar.com/warner-gpg.html
+
+
+-----BEGIN PGP SIGNATURE-----
+
+iQEzBAEBCgAdFiEE405i0G0Oac/KQXn/veDTHWhmanoFAmAHIyIACgkQveDTHWhm
+anqhqQf/YSbMXL+gwFhAZsjX39EVlbr/Ik7WPPkJW7v1oHybTnwFpFIc52COU1x/
+sqRfk4OyYtz9IBgOPXoWgXu9R4qdK6vYKxEsekcGT9C5l0OyDz8YWXEWgbGK5mvI
+aEub9WucD8r2uOQnnW6DtznFuEpvOjtf/+2BU767+bvLsbViW88ocbuLfCqLdOgD
+WZT9j3M+Y2Dc56DAJzP/4fkrUSVIofZStYp5u9HBjburgcYIp0g/cyc4xXRoi6Mp
+lFTRFv3MIjmoamzSQseoIgP6fi8QRqPrffPrsyqAp+06mJnPhxxFqxtO/ZErmpSa
+BGrLBxdWa8IF9U1A4Fs5nuAzAKMEg==
+=E9J+
+-----END PGP SIGNATURE-----
--- a/docs/frontends/FTP-and-SFTP.rst
+++ b/docs/frontends/FTP-and-SFTP.rst
@ -1,22 +1,21 @@
 .. -*- coding: utf-8-with-signature -*-

-=================================
-Tahoe-LAFS SFTP and FTP Frontends
-=================================
+========================
+Tahoe-LAFS SFTP Frontend
+========================

-1.  `SFTP/FTP Background`_
+1.  `SFTP Background`_
 2.  `Tahoe-LAFS Support`_
 3.  `Creating an Account File`_
 4.  `Running An Account Server (accounts.url)`_
 5.  `Configuring SFTP Access`_
-6.  `Configuring FTP Access`_
-7.  `Dependencies`_
-8.  `Immutable and Mutable Files`_
-9.  `Known Issues`_
+6.  `Dependencies`_
+7.  `Immutable and Mutable Files`_
+8.  `Known Issues`_


-SFTP/FTP Background
-===================
+SFTP Background
+===============

 FTP is the venerable internet file-transfer protocol, first developed in
 1971. The FTP server usually listens on port 21. A separate connection is
@ -33,20 +32,18 @@ Both FTP and SFTP were developed assuming a UNIX-like server, with accounts
 and passwords, octal file modes (user/group/other, read/write/execute), and
 ctime/mtime timestamps.

-We recommend SFTP over FTP, because the protocol is better, and the server
-implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below,
-for details.
+Previous versions of Tahoe-LAFS supported FTP, but now only the superior SFTP
+frontend is supported. See `Known Issues`_, below, for details on the
+limitations of SFTP.

 Tahoe-LAFS Support
 ==================

 All Tahoe-LAFS client nodes can run a frontend SFTP server, allowing regular
 SFTP clients (like ``/usr/bin/sftp``, the ``sshfs`` FUSE plugin, and many
-others) to access the file store. They can also run an FTP server, so FTP
-clients (like ``/usr/bin/ftp``, ``ncftp``, and others) can too. These
-frontends sit at the same level as the web-API interface.
+others) to access the file store.

-Since Tahoe-LAFS does not use user accounts or passwords, the SFTP/FTP
+Since Tahoe-LAFS does not use user accounts or passwords, the SFTP
 servers must be configured with a way to first authenticate a user (confirm
 that a prospective client has a legitimate claim to whatever authorities we
 might grant a particular user), and second to decide what directory cap
@ -173,39 +170,6 @@ clients and with the sshfs filesystem, see wiki:SftpFrontend_

 .. _wiki:SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend

-Configuring FTP Access
-======================
-
-To enable the FTP server with an accounts file, add the following lines to
-the BASEDIR/tahoe.cfg file::
-
- [ftpd]
- enabled = true
- port = tcp:8021:interface=127.0.0.1
- accounts.file = private/accounts
-
-The FTP server will listen on the given port number and on the loopback
-interface only. The "accounts.file" pathname will be interpreted relative to
-the node's BASEDIR.
-
-To enable the FTP server with an account server instead, provide the URL of
-that server in an "accounts.url" directive::
-
- [ftpd]
- enabled = true
- port = tcp:8021:interface=127.0.0.1
- accounts.url = https://example.com/login
-
-You can provide both accounts.file and accounts.url, although it probably
-isn't very useful except for testing.
-
-FTP provides no security, and so your password or caps could be eavesdropped
-if you connect to the FTP server remotely. The examples above include
-":interface=127.0.0.1" in the "port" option, which causes the server to only
-accept connections from localhost.
-
-Public key authentication is not supported for FTP.
-
 Dependencies
 ============

@ -216,7 +180,7 @@ separately: debian puts it in the "python-twisted-conch" package.
 Immutable and Mutable Files
 ===========================

-All files created via SFTP (and FTP) are immutable files. However, files can
+All files created via SFTP are immutable files. However, files can
 only be created in writeable directories, which allows the directory entry to
 be relinked to a different file. Normally, when the path of an immutable file
 is opened for writing by SFTP, the directory entry is relinked to another
@ -256,18 +220,3 @@ See also wiki:SftpFrontend_.

 .. _ticket #1059: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1059
 .. _ticket #1089: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1089
-
-Known Issues in the FTP Frontend
--------------------------------
-
-Mutable files are not supported by the FTP frontend (`ticket #680`_).
-
-Non-ASCII filenames are not supported by FTP (`ticket #682`_).
-
-The FTP frontend sometimes fails to report errors, for example if an upload
-fails because it does meet the "servers of happiness" threshold (`ticket
-#1081`_).
-
-.. _ticket #680: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/680
-.. _ticket #682: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/682
-.. _ticket #1081: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1081
--- a/docs/frontends/webapi.rst
+++ b/docs/frontends/webapi.rst
@ -2032,10 +2032,11 @@ potential for surprises when the file store structure is changed.

 Tahoe-LAFS provides a mutable file store, but the ways that the store can
 change are limited. The only things that can change are:
- * the mapping from child names to child objects inside mutable directories
-   (by adding a new child, removing an existing child, or changing an
-   existing child to point to a different object)
- * the contents of mutable files
+
+* the mapping from child names to child objects inside mutable directories
+  (by adding a new child, removing an existing child, or changing an
+  existing child to point to a different object)
+* the contents of mutable files

 Obviously if you query for information about the file store and then act
 to change it (such as by getting a listing of the contents of a mutable
@ -2157,7 +2158,7 @@ When modifying the file, be careful to update it atomically, otherwise a
 request may arrive while the file is only halfway written, and the partial
 file may be incorrectly parsed.

-The blacklist is applied to all access paths (including SFTP, FTP, and CLI
+The blacklist is applied to all access paths (including SFTP and CLI
 operations), not just the web-API. The blacklist also applies to directories.
 If a directory is blacklisted, the gateway will refuse access to both that
 directory and any child files/directories underneath it, when accessed via
--- a/docs/helper.rst
+++ b/docs/helper.rst
@ -122,7 +122,7 @@ Who should consider using a Helper?
 * clients who experience problems with TCP connection fairness: if other
  programs or machines in the same home are getting less than their fair
  share of upload bandwidth. If the connection is being shared fairly, then
-  a Tahoe upload that is happening at the same time as a single FTP upload
+  a Tahoe upload that is happening at the same time as a single SFTP upload
  should get half the bandwidth.
 * clients who have been given the helper.furl by someone who is running a
  Helper and is willing to let them use it
--- a/docs/index.rst
+++ b/docs/index.rst
@ -23,8 +23,9 @@ Contents:
   frontends/download-status

   known_issues
-   ../.github/CONTRIBUTING
+   contributing
   CODE_OF_CONDUCT
+   release-checklist

   servers
   helper
--- a/docs/known_issues.rst
+++ b/docs/known_issues.rst
@ -23,7 +23,7 @@ Known Issues in Tahoe-LAFS v1.10.3, released 30-Mar-2016
  *  `Disclosure of file through embedded hyperlinks or JavaScript in that file`_
  *  `Command-line arguments are leaked to other local users`_
  *  `Capabilities may be leaked to web browser phishing filter / "safe browsing" servers`_
-  *  `Known issues in the FTP and SFTP frontends`_
+  *  `Known issues in the SFTP frontend`_
  *  `Traffic analysis based on sizes of files/directories, storage indices, and timing`_
  *  `Privacy leak via Google Chart API link in map-update timing web page`_

@ -213,8 +213,8 @@ To disable the filter in Chrome:

 ----

-Known issues in the FTP and SFTP frontends
------------------------------------------
+Known issues in the SFTP frontend
+---------------------------------

 These are documented in :doc:`frontends/FTP-and-SFTP` and on `the
 SftpFrontend page`_ on the wiki.
--- a/docs/release-checklist.rst
+++ b/docs/release-checklist.rst
@ -40,23 +40,31 @@ Create Branch and Apply Updates
 - Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`)
 - run `tox -e news` to produce a new NEWS.txt file (this does a commit)
 - create the news for the release
+
  - newsfragments/<ticket number>.minor
  - commit it
+
 - manually fix NEWS.txt
+
  - proper title for latest release ("Release 1.15.0" instead of "Release ...post1432")
  - double-check date (maybe release will be in the future)
  - spot-check the release notes (these come from the newsfragments
    files though so don't do heavy editing)
  - commit these changes
+
 - update "relnotes.txt"
+
  - update all mentions of 1.14.0 -> 1.15.0
  - update "previous release" statement and date
  - summarize major changes
  - commit it
+
 - update "CREDITS"
+
  - are there any new contributors in this release?
  - one way: git log release-1.14.0.. | grep Author | sort | uniq
  - commit it
+
 - update "docs/known_issues.rst" if appropriate
 - update "docs/INSTALL.rst" references to the new release
 - Push the branch to github
@ -82,25 +90,36 @@ they will need to evaluate which contributors' signatures they trust.

 - (all steps above are completed)
 - sign the release
+
  - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0
  - (replace the key-id above with your own)
+
 - build all code locally
  - these should all pass:
+
    - tox -e py27,codechecks,docs,integration
+
  - these can fail (ideally they should not of course):
+
    - tox -e deprecations,upcoming-deprecations
+
 - build tarballs
+
  - tox -e tarballs
  - confirm it at least exists:
  - ls dist/ | grep 1.15.0rc0
+
 - inspect and test the tarballs
+
  - install each in a fresh virtualenv
  - run `tahoe` command
+
 - when satisfied, sign the tarballs:
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.zip
+
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip


 Privileged Contributor
@ -118,6 +137,12 @@ Did anyone contribute a hack since the last release? If so, then
 https://tahoe-lafs.org/hacktahoelafs/ needs to be updated.


+Sign Git Tag
+````````````
+
+- git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
+
+
 Upload Artifacts
 ````````````````

@ -129,6 +154,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads`
  https://tahoe-lafs.org/downloads/ on the Web.
 - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads
 - the following developers have access to do this:
+
  - exarkun
  - meejah
  - warner
@ -137,8 +163,9 @@ For the actual release, the tarball and signature files need to be
 uploaded to PyPI as well.

 - how to do this?
- (original guide says only "twine upload dist/*")
+- (original guide says only `twine upload dist/*`)
 - the following developers have access to do this:
+
  - warner
  - exarkun (partial?)
  - meejah (partial?)
--- a/docs/running.rst
+++ b/docs/running.rst
@ -207,10 +207,10 @@ create a new directory and lose the capability to it, then you cannot
 access that directory ever again.


-The SFTP and FTP frontends
--------------------------
+The SFTP frontend
+-----------------

-You can access your Tahoe-LAFS grid via any SFTP_ or FTP_ client. See
+You can access your Tahoe-LAFS grid via any SFTP_ client. See
 :doc:`frontends/FTP-and-SFTP` for how to set this up. On most Unix
 platforms, you can also use SFTP to plug Tahoe-LAFS into your computer's
 local filesystem via ``sshfs``, but see the `FAQ about performance
@ -220,7 +220,6 @@ The SftpFrontend_ page on the wiki has more information about using SFTP with
 Tahoe-LAFS.

 .. _SFTP:  https://en.wikipedia.org/wiki/SSH_file_transfer_protocol
-.. _FTP: https://en.wikipedia.org/wiki/File_Transfer_Protocol
 .. _FAQ about performance problems: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/FAQ#Q23_FUSE
 .. _SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend

--- a/integration/conftest.py
+++ b/integration/conftest.py
@ -7,6 +7,7 @@ from os import mkdir, listdir, environ
 from os.path import join, exists
 from tempfile import mkdtemp, mktemp
 from functools import partial
+from json import loads

 from foolscap.furl import (
    decode_furl,
@ -37,6 +38,10 @@ from util import (
    _tahoe_runner_optional_coverage,
    await_client_ready,
    TahoeProcess,
+    cli,
+    _run_node,
+    generate_ssh_key,
+    block_with_timeout,
 )


@ -152,7 +157,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
        )
        print("Waiting for flogtool to complete")
        try:
-            pytest_twisted.blockon(flog_protocol.done)
+            block_with_timeout(flog_protocol.done, reactor)
        except ProcessTerminated as e:
            print("flogtool exited unexpectedly: {}".format(str(e)))
        print("Flogtool completed")
@ -293,7 +298,7 @@ log_gatherer.furl = {log_furl}
    def cleanup():
        try:
            transport.signalProcess('TERM')
-            pytest_twisted.blockon(protocol.exited)
+            block_with_timeout(protocol.exited, reactor)
        except ProcessExitedAlready:
            pass
    request.addfinalizer(cleanup)
@ -347,8 +352,50 @@ def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, requ
            reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
            web_port="tcp:9980:interface=localhost",
            storage=False,
+            # We're going to kill this ourselves, so no need for finalizer to
+            # do it:
+            finalize=False,
        )
    )
+    await_client_ready(process)
+
+    # 1. Create a new RW directory cap:
+    cli(process, "create-alias", "test")
+    rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
+
+    # 2. Enable SFTP on the node:
+    host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
+    accounts_path = join(process.node_dir, "private", "accounts")
+    with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
+        f.write("""\
+[sftpd]
+enabled = true
+port = tcp:8022:interface=127.0.0.1
+host_pubkey_file = {ssh_key_path}.pub
+host_privkey_file = {ssh_key_path}
+accounts.file = {accounts_path}
+""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
+    generate_ssh_key(host_ssh_key_path)
+
+    # 3. Add a SFTP access file with username/password and SSH key auth.
+
+    # The client SSH key path is typically going to be somewhere else (~/.ssh,
+    # typically), but for convenience sake for testing we'll put it inside node.
+    client_ssh_key_path = join(process.node_dir, "private", "ssh_client_rsa_key")
+    generate_ssh_key(client_ssh_key_path)
+    # Pub key format is "ssh-rsa <thekey> <username>". We want the key.
+    ssh_public_key = open(client_ssh_key_path + ".pub").read().strip().split()[1]
+    with open(accounts_path, "w") as f:
+        f.write("""\
+alice password {rwcap}
+
+alice2 ssh-rsa {ssh_public_key} {rwcap}
+""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
+
+    # 4. Restart the node with new SFTP config.
+    process.kill()
+    pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
+
    await_client_ready(process)
    return process

@ -490,7 +537,13 @@ def tor_network(reactor, temp_dir, chutney, request):
            path=join(chutney_dir),
            env=env,
        )
-        pytest_twisted.blockon(proto.done)
+        try:
+            block_with_timeout(proto.done, reactor)
+        except ProcessTerminated:
+            # If this doesn't exit cleanly, that's fine, that shouldn't fail
+            # the test suite.
+            pass
+
    request.addfinalizer(cleanup)

    return chut
--- a/integration/test_sftp.py
+++ b/integration/test_sftp.py
@ -0,0 +1,162 @@
+"""
+It's possible to create/rename/delete files and directories in Tahoe-LAFS using
+SFTP.
+
+These tests use Paramiko, rather than Twisted's Conch, because:
+
+    1. It's a different implementation, so we're not testing Conch against
+       itself.
+
+    2. Its API is much simpler to use.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from posixpath import join
+from stat import S_ISDIR
+
+from paramiko import SSHClient
+from paramiko.client import AutoAddPolicy
+from paramiko.sftp_client import SFTPClient
+from paramiko.ssh_exception import AuthenticationException
+from paramiko.rsakey import RSAKey
+
+import pytest
+
+from .util import generate_ssh_key, run_in_thread
+
+
+def connect_sftp(connect_args={"username": "alice", "password": "password"}):
+    """Create an SFTP client."""
+    client = SSHClient()
+    client.set_missing_host_key_policy(AutoAddPolicy)
+    client.connect("localhost", port=8022, look_for_keys=False,
+                   allow_agent=False, **connect_args)
+    sftp = SFTPClient.from_transport(client.get_transport())
+
+    def rmdir(path, delete_root=True):
+        for f in sftp.listdir_attr(path=path):
+            childpath = join(path, f.filename)
+            if S_ISDIR(f.st_mode):
+                rmdir(childpath)
+            else:
+                sftp.remove(childpath)
+        if delete_root:
+            sftp.rmdir(path)
+
+    # Delete any files left over from previous tests :(
+    rmdir("/", delete_root=False)
+
+    return sftp
+
+
+@run_in_thread
+def test_bad_account_password_ssh_key(alice, tmpdir):
+    """
+    Can't login with unknown username, wrong password, or wrong SSH pub key.
+    """
+    # Wrong password, wrong username:
+    for u, p in [("alice", "wrong"), ("someuser", "password")]:
+        with pytest.raises(AuthenticationException):
+            connect_sftp(connect_args={
+                "username": u, "password": p,
+            })
+
+    another_key = join(str(tmpdir), "ssh_key")
+    generate_ssh_key(another_key)
+    good_key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key"))
+    bad_key = RSAKey(filename=another_key)
+
+    # Wrong key:
+    with pytest.raises(AuthenticationException):
+        connect_sftp(connect_args={
+            "username": "alice2", "pkey": bad_key,
+        })
+
+    # Wrong username:
+    with pytest.raises(AuthenticationException):
+        connect_sftp(connect_args={
+            "username": "someoneelse", "pkey": good_key,
+        })
+
+
+@run_in_thread
+def test_ssh_key_auth(alice):
+    """It's possible to login authenticating with SSH public key."""
+    key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key"))
+    sftp = connect_sftp(connect_args={
+        "username": "alice2", "pkey": key
+    })
+    assert sftp.listdir() == []
+
+
+@run_in_thread
+def test_read_write_files(alice):
+    """It's possible to upload and download files."""
+    sftp = connect_sftp()
+    with sftp.file("myfile", "wb") as f:
+        f.write(b"abc")
+        f.write(b"def")
+
+    with sftp.file("myfile", "rb") as f:
+        assert f.read(4) == b"abcd"
+        assert f.read(2) == b"ef"
+        assert f.read(1) == b""
+
+
+@run_in_thread
+def test_directories(alice):
+    """
+    It's possible to create, list directories, and create and remove files in
+    them.
+    """
+    sftp = connect_sftp()
+    assert sftp.listdir() == []
+
+    sftp.mkdir("childdir")
+    assert sftp.listdir() == ["childdir"]
+
+    with sftp.file("myfile", "wb") as f:
+        f.write(b"abc")
+    assert sorted(sftp.listdir()) == ["childdir", "myfile"]
+
+    sftp.chdir("childdir")
+    assert sftp.listdir() == []
+
+    with sftp.file("myfile2", "wb") as f:
+        f.write(b"def")
+    assert sftp.listdir() == ["myfile2"]
+
+    sftp.chdir(None)  # root
+    with sftp.file("childdir/myfile2", "rb") as f:
+        assert f.read() == b"def"
+
+    sftp.remove("myfile")
+    assert sftp.listdir() == ["childdir"]
+
+    sftp.rmdir("childdir")
+    assert sftp.listdir() == []
+
+
+@run_in_thread
+def test_rename(alice):
+    """Directories and files can be renamed."""
+    sftp = connect_sftp()
+    sftp.mkdir("dir")
+
+    filepath = join("dir", "file")
+    with sftp.file(filepath, "wb") as f:
+        f.write(b"abc")
+
+    sftp.rename(filepath, join("dir", "file2"))
+    sftp.rename("dir", "dir2")
+
+    with sftp.file(join("dir2", "file2"), "rb") as f:
+        assert f.read() == b"abc"
--- a/integration/test_web.py
+++ b/integration/test_web.py
@ -127,12 +127,12 @@ def test_deep_stats(alice):
        dircap_uri,
        data={
            u"t": u"upload",
-            u"when_done": u".",
        },
        files={
            u"file": FILE_CONTENTS,
        },
    )
+    resp.raise_for_status()

    # confirm the file is in the directory
    resp = requests.get(
@ -175,6 +175,7 @@ def test_deep_stats(alice):
        time.sleep(.5)


+@util.run_in_thread
 def test_status(alice):
    """
    confirm we get something sensible from /status and the various sub-types
--- a/integration/util.py
+++ b/integration/util.py
@ -5,6 +5,7 @@ from os import mkdir, environ
 from os.path import exists, join
 from six.moves import StringIO
 from functools import partial
+from subprocess import check_output

 from twisted.python.filepath import (
    FilePath,
@ -12,9 +13,13 @@ from twisted.python.filepath import (
 from twisted.internet.defer import Deferred, succeed
 from twisted.internet.protocol import ProcessProtocol
 from twisted.internet.error import ProcessExitedAlready, ProcessDone
+from twisted.internet.threads import deferToThread

 import requests

+from paramiko.rsakey import RSAKey
+from boltons.funcutils import wraps
+
 from allmydata.util.configutil import (
    get_config,
    set_config,
@ -25,6 +30,12 @@ from allmydata import client
 import pytest_twisted


+def block_with_timeout(deferred, reactor, timeout=120):
+    """Block until Deferred has result, but timeout instead of waiting forever."""
+    deferred.addTimeout(timeout, reactor)
+    return pytest_twisted.blockon(deferred)
+
+
 class _ProcessExitedProtocol(ProcessProtocol):
    """
    Internal helper that .callback()s on self.done when the process
@ -123,11 +134,12 @@ def _cleanup_tahoe_process(tahoe_transport, exited):

    :return: After the process has exited.
    """
+    from twisted.internet import reactor
    try:
        print("signaling {} with TERM".format(tahoe_transport.pid))
        tahoe_transport.signalProcess('TERM')
        print("signaled, blocking on exit")
-        pytest_twisted.blockon(exited)
+        block_with_timeout(exited, reactor)
        print("exited, goodbye")
    except ProcessExitedAlready:
        pass
@ -175,11 +187,15 @@ class TahoeProcess(object):
            u"portnum",
        )

+    def kill(self):
+        """Kill the process, block until it's done."""
+        _cleanup_tahoe_process(self.transport, self.transport.exited)
+
    def __str__(self):
        return "<TahoeProcess in '{}'>".format(self._node_dir)


-def _run_node(reactor, node_dir, request, magic_text):
+def _run_node(reactor, node_dir, request, magic_text, finalize=True):
    """
    Run a tahoe process from its node_dir.

@ -203,7 +219,8 @@ def _run_node(reactor, node_dir, request, magic_text):
    )
    transport.exited = protocol.exited

-    request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
+    if finalize:
+        request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))

    # XXX abusing the Deferred; should use .when_magic_seen() pattern

@ -222,7 +239,8 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
                 magic_text=None,
                 needed=2,
                 happy=3,
-                 total=4):
+                 total=4,
+                 finalize=True):
    """
    Helper to create a single node, run it and return the instance
    spawnProcess returned (ITransport)
@ -270,7 +288,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
    d = Deferred()
    d.callback(None)
    d.addCallback(lambda _: created_d)
-    d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text))
+    d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text, finalize=finalize))
    return d


@ -390,17 +408,13 @@ def await_file_vanishes(path, timeout=10):
    raise FileShouldVanishException(path, timeout)


-def cli(request, reactor, node_dir, *argv):
+def cli(node, *argv):
    """
-    Run a tahoe CLI subcommand for a given node, optionally running
-    under coverage if '--coverage' was supplied.
+    Run a tahoe CLI subcommand for a given node in a blocking manner, returning
+    the output.
    """
-    proto = _CollectOutputProtocol()
-    _tahoe_runner_optional_coverage(
-        proto, reactor, request,
-        ['--node-directory', node_dir] + list(argv),
-    )
-    return proto.done
+    arguments = ["tahoe", '--node-directory', node.node_dir]
+    return check_output(arguments + list(argv))


 def node_url(node_dir, uri_fragment):
@ -505,3 +519,36 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2):
            tahoe,
        )
    )
+
+
+def generate_ssh_key(path):
+    """Create a new SSH private/public key pair."""
+    key = RSAKey.generate(2048)
+    key.write_private_key_file(path)
+    with open(path + ".pub", "wb") as f:
+        f.write(b"%s %s" % (key.get_name(), key.get_base64()))
+
+
+def run_in_thread(f):
+    """Decorator for integration tests that runs code in a thread.
+
+    Because we're using pytest_twisted, tests that rely on the reactor are
+    expected to return a Deferred and use async APIs so the reactor can run.
+
+    In the case of the integration test suite, it launches nodes in the
+    background using Twisted APIs.  The nodes stdout and stderr is read via
+    Twisted code.  If the reactor doesn't run, reads don't happen, and
+    eventually the buffers fill up, and the nodes block when they try to flush
+    logs.
+
+    We can switch to Twisted APIs (treq instead of requests etc.), but
+    sometimes it's easier or expedient to just have a blocking test.  So this
+    decorator allows you to run the test in a thread, and the reactor can keep
+    running in the main thread.
+
+    See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
+    """
+    @wraps(f)
+    def test(*args, **kwargs):
+        return deferToThread(lambda: f(*args, **kwargs))
+    return test
--- a/misc/operations_helpers/provisioning/provisioning.py
+++ b/misc/operations_helpers/provisioning/provisioning.py
@ -46,7 +46,7 @@ class ProvisioningTool(rend.Page):
        req = inevow.IRequest(ctx)

        def getarg(name, astype=int):
-            if req.method != "POST":
+            if req.method != b"POST":
                return None
            if name in req.fields:
                return astype(req.fields[name].value)
--- a/mypy.ini
+++ b/mypy.ini
@ -0,0 +1,3 @@
+[mypy]
+ignore_missing_imports = True
+plugins=mypy_zope:plugin
--- a/newsfragments/3326.installation
+++ b/newsfragments/3326.installation
@ -0,0 +1 @@
+Debian 8 support has been replaced with Debian 10 support.
--- a/newsfragments/3384.minor
+++ b/newsfragments/3384.minor
--- a/newsfragments/3385.minor
+++ b/newsfragments/3385.minor
--- a/newsfragments/3399.feature
+++ b/newsfragments/3399.feature
@ -0,0 +1 @@
+Added 'typechecks' environment for tox running mypy and performing static typechecks.
--- a/newsfragments/3529.minor
+++ b/newsfragments/3529.minor
--- a/newsfragments/3534.minor
+++ b/newsfragments/3534.minor
--- a/newsfragments/3536.minor
+++ b/newsfragments/3536.minor
--- a/newsfragments/3552.minor
+++ b/newsfragments/3552.minor
--- a/newsfragments/3564.minor
+++ b/newsfragments/3564.minor
--- a/newsfragments/3565.minor
+++ b/newsfragments/3565.minor
--- a/newsfragments/3566.minor
+++ b/newsfragments/3566.minor
--- a/newsfragments/3567.minor
+++ b/newsfragments/3567.minor
--- a/newsfragments/3568.minor
+++ b/newsfragments/3568.minor
--- a/newsfragments/3572.minor
+++ b/newsfragments/3572.minor
--- a/newsfragments/3574.minor
+++ b/newsfragments/3574.minor
--- a/newsfragments/3575.minor
+++ b/newsfragments/3575.minor
--- a/newsfragments/3576.minor
+++ b/newsfragments/3576.minor
--- a/newsfragments/3577.minor
+++ b/newsfragments/3577.minor
--- a/newsfragments/3578.minor
+++ b/newsfragments/3578.minor
--- a/newsfragments/3579.minor
+++ b/newsfragments/3579.minor
--- a/newsfragments/3580.minor
+++ b/newsfragments/3580.minor
--- a/newsfragments/3582.minor
+++ b/newsfragments/3582.minor
--- a/newsfragments/3583.removed
+++ b/newsfragments/3583.removed
@ -0,0 +1 @@
+FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead.
--- a/newsfragments/3584.bugfix
+++ b/newsfragments/3584.bugfix
@ -0,0 +1 @@
+SFTP public key auth likely works more consistently, and SFTP in general was previously broken.
--- a/newsfragments/3587.minor
+++ b/newsfragments/3587.minor
@ -0,0 +1 @@
+
--- a/newsfragments/3588.incompat
+++ b/newsfragments/3588.incompat
@ -0,0 +1 @@
+The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale.
--- a/newsfragments/3588.minor
+++ b/newsfragments/3588.minor
--- a/newsfragments/3589.minor
+++ b/newsfragments/3589.minor
--- a/newsfragments/3590.bugfix
+++ b/newsfragments/3590.bugfix
@ -0,0 +1 @@
+Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work.
--- a/newsfragments/3591.minor
+++ b/newsfragments/3591.minor
--- a/newsfragments/3592.minor
+++ b/newsfragments/3592.minor
--- a/newsfragments/3593.minor
+++ b/newsfragments/3593.minor
--- a/newsfragments/3594.minor
+++ b/newsfragments/3594.minor
--- a/newsfragments/3595.minor
+++ b/newsfragments/3595.minor
--- a/newsfragments/3596.minor
+++ b/newsfragments/3596.minor
--- a/newsfragments/3599.minor
+++ b/newsfragments/3599.minor
--- a/newsfragments/3600.minor
+++ b/newsfragments/3600.minor
--- a/newsfragments/3612.minor
+++ b/newsfragments/3612.minor
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@ -23,17 +23,12 @@ python.pkgs.buildPythonPackage rec {
    # This list is over-zealous because it's more work to disable individual
    # tests with in a module.

-    # test_system is a lot of integration-style tests that do a lot of real
-    # networking between many processes.  They sometimes fail spuriously.
-    rm src/allmydata/test/test_system.py
-
    # Many of these tests don't properly skip when i2p or tor dependencies are
    # not supplied (and we are not supplying them).
    rm src/allmydata/test/test_i2p_provider.py
    rm src/allmydata/test/test_connections.py
    rm src/allmydata/test/cli/test_create.py
    rm src/allmydata/test/test_client.py
-    rm src/allmydata/test/test_runner.py
  '';


--- a/setup.py
+++ b/setup.py
@ -63,12 +63,8 @@ install_requires = [
    #   version of cryptography will *really* be installed.
    "cryptography >= 2.6",

-    # * We need Twisted 10.1.0 for the FTP frontend in order for
-    #   Twisted's FTP server to support asynchronous close.
    # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
    #   rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
-    # * The FTP frontend depends on Twisted >= 11.1.0 for
-    #   filepath.Permissions
    # * The SFTP frontend and manhole depend on the conch extra. However, we
    #   can't explicitly declare that without an undesirable dependency on gmpy,
    #   as explained in ticket #2740.
@ -111,7 +107,9 @@ install_requires = [

    # Eliot is contemplating dropping Python 2 support.  Stick to a version we
    # know works on Python 2.7.
-    "eliot ~= 1.7",
+    "eliot ~= 1.7 ; python_version < '3.0'",
+    # On Python 3, we want a new enough version to support custom JSON encoders.
+    "eliot >= 1.13.0 ; python_version > '3.0'",

    # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped
    # Python 2 entirely; stick to the version known to work for us.
@ -383,10 +381,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
              # this version from time to time, but we will do it
              # intentionally.
              "pyflakes == 2.2.0",
-              # coverage 5.0 breaks the integration tests in some opaque way.
-              # This probably needs to be addressed in a more permanent way
-              # eventually...
-              "coverage ~= 4.5",
+              "coverage ~= 5.0",
              "mock",
              "tox",
              "pytest",
@ -400,6 +395,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
              "html5lib",
              "junitxml",
              "tenacity",
+              "paramiko",
+              "pytest-timeout",
          ] + tor_requires + i2p_requires,
          "tor": tor_requires,
          "i2p": i2p_requires,
--- a/src/allmydata/init.py
+++ b/src/allmydata/init.py
@ -14,7 +14,9 @@ __all__ = [

 __version__ = "unknown"
 try:
-    from allmydata._version import __version__
+    # type ignored as it fails in CI
+    # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972)
+    from allmydata._version import __version__  # type: ignore
 except ImportError:
    # We're running in a tree that hasn't run update_version, and didn't
    # come with a _version.py, so we don't know what our version is.
@ -24,7 +26,9 @@ except ImportError:
 full_version = "unknown"
 branch = "unknown"
 try:
-    from allmydata._version import full_version, branch
+    # type ignored as it fails in CI
+    # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972)
+    from allmydata._version import full_version, branch  # type: ignore
 except ImportError:
    # We're running in a tree that hasn't run update_version, and didn't
    # come with a _version.py, so we don't know what our full version or
--- a/src/allmydata/blacklist.py
+++ b/src/allmydata/blacklist.py
@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+        from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

 import os

@ -34,10 +45,10 @@ class Blacklist(object):
        try:
            if self.last_mtime is None or current_mtime > self.last_mtime:
                self.entries.clear()
-                with open(self.blacklist_fn, "r") as f:
+                with open(self.blacklist_fn, "rb") as f:
                    for line in f:
                        line = line.strip()
-                        if not line or line.startswith("#"):
+                        if not line or line.startswith(b"#"):
                            continue
                        si_s, reason = line.split(None, 1)
                        si = base32.a2b(si_s) # must be valid base32
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@ -86,12 +86,6 @@ _client_config = configutil.ValidConfiguration(
            "shares.total",
            "storage.plugins",
        ),
-        "ftpd": (
-            "accounts.file",
-            "accounts.url",
-            "enabled",
-            "port",
-        ),
        "storage": (
            "debug_discard",
            "enabled",
@ -656,7 +650,6 @@ class _Client(node.Node, pollmixin.PollMixin):
                raise ValueError("config error: helper is enabled, but tub "
                                 "is not listening ('tub.port=' is empty)")
            self.init_helper()
-        self.init_ftp_server()
        self.init_sftp_server()

        # If the node sees an exit_trigger file, it will poll every second to see
@ -714,7 +707,7 @@ class _Client(node.Node, pollmixin.PollMixin):
    def get_long_nodeid(self):
        # this matches what IServer.get_longname() says about us elsewhere
        vk_string = ed25519.string_from_verifying_key(self._node_public_key)
-        return remove_prefix(vk_string, "pub-")
+        return remove_prefix(vk_string, b"pub-")

    def get_long_tubid(self):
        return idlib.nodeid_b2a(self.nodeid)
@ -898,10 +891,6 @@ class _Client(node.Node, pollmixin.PollMixin):
        if helper_furl in ("None", ""):
            helper_furl = None

-        # FURLs need to be bytes:
-        if helper_furl is not None:
-            helper_furl = helper_furl.encode("utf-8")
-
        DEP = self.encoding_params
        DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
        DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))
@ -1036,18 +1025,6 @@ class _Client(node.Node, pollmixin.PollMixin):
        )
        ws.setServiceParent(self)

-    def init_ftp_server(self):
-        if self.config.get_config("ftpd", "enabled", False, boolean=True):
-            accountfile = self.config.get_config("ftpd", "accounts.file", None)
-            if accountfile:
-                accountfile = self.config.get_config_path(accountfile)
-            accounturl = self.config.get_config("ftpd", "accounts.url", None)
-            ftp_portstr = self.config.get_config("ftpd", "port", "8021")
-
-            from allmydata.frontends import ftpd
-            s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr)
-            s.setServiceParent(self)
-
    def init_sftp_server(self):
        if self.config.get_config("sftpd", "enabled", False, boolean=True):
            accountfile = self.config.get_config("sftpd", "accounts.file", None)
--- a/src/allmydata/codec.py
+++ b/src/allmydata/codec.py
@ -57,6 +57,10 @@ class CRSEncoder(object):

        return defer.succeed((shares, desired_share_ids))

+    def encode_proposal(self, data, desired_share_ids=None):
+        raise NotImplementedError()
+
+
@implementer(ICodecDecoder)
 class CRSDecoder(object):

--- a/src/allmydata/deep_stats.py
+++ b/src/allmydata/deep_stats.py
@ -1,4 +1,15 @@
-"""Implementation of the deep stats class."""
+"""Implementation of the deep stats class.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401

 import math

@ -13,7 +24,7 @@ from allmydata.util import mathutil
 class DeepStats(object):
    """Deep stats object.

-    Holds results of the deep-stats opetation.
+    Holds results of the deep-stats operation.
    Used for json generation in the API."""

    # Json API version.
@ -121,7 +132,7 @@ class DeepStats(object):
        h[bucket] += 1

    def get_results(self):
-        """Returns deep-stats resutls."""
+        """Returns deep-stats results."""
        stats = self.stats.copy()
        for key in self.histograms:
            h = self.histograms[key]
--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@ -18,7 +18,6 @@ import time
 from zope.interface import implementer
 from twisted.internet import defer
 from foolscap.api import fireEventually
-import json

 from allmydata.crypto import aes
 from allmydata.deep_stats import DeepStats
@ -31,7 +30,7 @@ from allmydata.interfaces import IFilesystemNode, IDirectoryNode, IFileNode, \
 from allmydata.check_results import DeepCheckResults, \
     DeepCheckAndRepairResults
 from allmydata.monitor import Monitor
-from allmydata.util import hashutil, base32, log
+from allmydata.util import hashutil, base32, log, jsonbytes as json
 from allmydata.util.encodingutil import quote_output, normalize
 from allmydata.util.assertutil import precondition
 from allmydata.util.netstring import netstring, split_netstring
@ -569,7 +568,7 @@ class DirectoryNode(object):
        d = self.get_child_and_metadata(childnamex)
        return d

-    def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True):
+    def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True):
        precondition(isinstance(writecap, (bytes, type(None))), writecap)
        precondition(isinstance(readcap, (bytes, type(None))), readcap)

--- a/src/allmydata/frontends/auth.py
+++ b/src/allmydata/frontends/auth.py
@ -4,8 +4,8 @@ from zope.interface import implementer
 from twisted.web.client import getPage
 from twisted.internet import defer
 from twisted.cred import error, checkers, credentials
-from twisted.conch import error as conch_error
 from twisted.conch.ssh import keys
+from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB

 from allmydata.util import base32
 from allmydata.util.fileutil import abspath_expanduser_unicode
@ -29,7 +29,7 @@ class AccountFileChecker(object):
    def __init__(self, client, accountfile):
        self.client = client
        self.passwords = {}
-        self.pubkeys = {}
+        pubkeys = {}
        self.rootcaps = {}
        with open(abspath_expanduser_unicode(accountfile), "r") as f:
            for line in f:
@ -40,12 +40,14 @@ class AccountFileChecker(object):
                if passwd.startswith("ssh-"):
                    bits = rest.split()
                    keystring = " ".join([passwd] + bits[:-1])
+                    key = keys.Key.fromString(keystring)
                    rootcap = bits[-1]
-                    self.pubkeys[name] = keystring
+                    pubkeys[name] = [key]
                else:
                    self.passwords[name] = passwd
                    rootcap = rest
                self.rootcaps[name] = rootcap
+        self._pubkeychecker = SSHPublicKeyChecker(InMemorySSHKeyDB(pubkeys))

    def _avatarId(self, username):
        return FTPAvatarID(username, self.rootcaps[username])
@ -57,11 +59,9 @@ class AccountFileChecker(object):

    def requestAvatarId(self, creds):
        if credentials.ISSHPrivateKey.providedBy(creds):
-            # Re-using twisted.conch.checkers.SSHPublicKeyChecker here, rather
-            # than re-implementing all of the ISSHPrivateKey checking logic,
-            # would be better.  That would require Twisted 14.1.0 or newer,
-            # though.
-            return self._checkKey(creds)
+            d = defer.maybeDeferred(self._pubkeychecker.requestAvatarId, creds)
+            d.addCallback(self._avatarId)
+            return d
        elif credentials.IUsernameHashedPassword.providedBy(creds):
            return self._checkPassword(creds)
        elif credentials.IUsernamePassword.providedBy(creds):
@ -86,28 +86,6 @@ class AccountFileChecker(object):
        d.addCallback(self._cbPasswordMatch, str(creds.username))
        return d

-    def _checkKey(self, creds):
-        """
-        Determine whether some key-based credentials correctly authenticates a
-        user.
-
-        Returns a Deferred that fires with the username if so or with an
-        UnauthorizedLogin failure otherwise.
-        """
-
-        # Is the public key indicated by the given credentials allowed to
-        # authenticate the username in those credentials?
-        if creds.blob == self.pubkeys.get(creds.username):
-            if creds.signature is None:
-                return defer.fail(conch_error.ValidPublicKey())
-
-            # Is the signature in the given credentials the correct
-            # signature for the data in those credentials?
-            key = keys.Key.fromString(creds.blob)
-            if key.verify(creds.signature, creds.sigData):
-                return defer.succeed(self._avatarId(creds.username))
-
-        return defer.fail(error.UnauthorizedLogin())

@implementer(checkers.ICredentialsChecker)
 class AccountURLChecker(object):
--- a/src/allmydata/frontends/ftpd.py
+++ b/src/allmydata/frontends/ftpd.py
@ -1,340 +0,0 @@
-from six import ensure_str
-
-from types import NoneType
-
-from zope.interface import implementer
-from twisted.application import service, strports
-from twisted.internet import defer
-from twisted.internet.interfaces import IConsumer
-from twisted.cred import portal
-from twisted.python import filepath
-from twisted.protocols import ftp
-
-from allmydata.interfaces import IDirectoryNode, ExistingChildError, \
-     NoSuchChildError
-from allmydata.immutable.upload import FileHandle
-from allmydata.util.fileutil import EncryptedTemporaryFile
-from allmydata.util.assertutil import precondition
-
-@implementer(ftp.IReadFile)
-class ReadFile(object):
-    def __init__(self, node):
-        self.node = node
-    def send(self, consumer):
-        d = self.node.read(consumer)
-        return d # when consumed
-
-@implementer(IConsumer)
-class FileWriter(object):
-
-    def registerProducer(self, producer, streaming):
-        if not streaming:
-            raise NotImplementedError("Non-streaming producer not supported.")
-        # we write the data to a temporary file, since Tahoe can't do
-        # streaming upload yet.
-        self.f = EncryptedTemporaryFile()
-        return None
-
-    def unregisterProducer(self):
-        # the upload actually happens in WriteFile.close()
-        pass
-
-    def write(self, data):
-        self.f.write(data)
-
-@implementer(ftp.IWriteFile)
-class WriteFile(object):
-
-    def __init__(self, parent, childname, convergence):
-        self.parent = parent
-        self.childname = childname
-        self.convergence = convergence
-
-    def receive(self):
-        self.c = FileWriter()
-        return defer.succeed(self.c)
-
-    def close(self):
-        u = FileHandle(self.c.f, self.convergence)
-        d = self.parent.add_file(self.childname, u)
-        return d
-
-
-class NoParentError(Exception):
-    pass
-
-# filepath.Permissions was added in Twisted-11.1.0, which we require. Twisted
-# <15.0.0 expected an int, and only does '&' on it. Twisted >=15.0.0 expects
-# a filepath.Permissions. This satisfies both.
-
-class IntishPermissions(filepath.Permissions):
-    def __init__(self, statModeInt):
-        self._tahoe_statModeInt = statModeInt
-        filepath.Permissions.__init__(self, statModeInt)
-    def __and__(self, other):
-        return self._tahoe_statModeInt & other
-
-@implementer(ftp.IFTPShell)
-class Handler(object):
-    def __init__(self, client, rootnode, username, convergence):
-        self.client = client
-        self.root = rootnode
-        self.username = username
-        self.convergence = convergence
-
-    def makeDirectory(self, path):
-        d = self._get_root(path)
-        d.addCallback(lambda root_and_path:
-                      self._get_or_create_directories(root_and_path[0], root_and_path[1]))
-        return d
-
-    def _get_or_create_directories(self, node, path):
-        if not IDirectoryNode.providedBy(node):
-            # unfortunately it is too late to provide the name of the
-            # blocking directory in the error message.
-            raise ftp.FileExistsError("cannot create directory because there "
-                                      "is a file in the way")
-        if not path:
-            return defer.succeed(node)
-        d = node.get(path[0])
-        def _maybe_create(f):
-            f.trap(NoSuchChildError)
-            return node.create_subdirectory(path[0])
-        d.addErrback(_maybe_create)
-        d.addCallback(self._get_or_create_directories, path[1:])
-        return d
-
-    def _get_parent(self, path):
-        # fire with (parentnode, childname)
-        path = [unicode(p) for p in path]
-        if not path:
-            raise NoParentError
-        childname = path[-1]
-        d = self._get_root(path)
-        def _got_root(root_and_path):
-            (root, path) = root_and_path
-            if not path:
-                raise NoParentError
-            return root.get_child_at_path(path[:-1])
-        d.addCallback(_got_root)
-        def _got_parent(parent):
-            return (parent, childname)
-        d.addCallback(_got_parent)
-        return d
-
-    def _remove_thing(self, path, must_be_directory=False, must_be_file=False):
-        d = defer.maybeDeferred(self._get_parent, path)
-        def _convert_error(f):
-            f.trap(NoParentError)
-            raise ftp.PermissionDeniedError("cannot delete root directory")
-        d.addErrback(_convert_error)
-        def _got_parent(parent_and_childname):
-            (parent, childname) = parent_and_childname
-            d = parent.get(childname)
-            def _got_child(child):
-                if must_be_directory and not IDirectoryNode.providedBy(child):
-                    raise ftp.IsNotADirectoryError("rmdir called on a file")
-                if must_be_file and IDirectoryNode.providedBy(child):
-                    raise ftp.IsADirectoryError("rmfile called on a directory")
-                return parent.delete(childname)
-            d.addCallback(_got_child)
-            d.addErrback(self._convert_error)
-            return d
-        d.addCallback(_got_parent)
-        return d
-
-    def removeDirectory(self, path):
-        return self._remove_thing(path, must_be_directory=True)
-
-    def removeFile(self, path):
-        return self._remove_thing(path, must_be_file=True)
-
-    def rename(self, fromPath, toPath):
-        # the target directory must already exist
-        d = self._get_parent(fromPath)
-        def _got_from_parent(fromparent_and_childname):
-            (fromparent, childname) = fromparent_and_childname
-            d = self._get_parent(toPath)
-            d.addCallback(lambda toparent_and_tochildname:
-                          fromparent.move_child_to(childname,
-                                                   toparent_and_tochildname[0], toparent_and_tochildname[1],
-                                                   overwrite=False))
-            return d
-        d.addCallback(_got_from_parent)
-        d.addErrback(self._convert_error)
-        return d
-
-    def access(self, path):
-        # we allow access to everything that exists. We are required to raise
-        # an error for paths that don't exist: FTP clients (at least ncftp)
-        # uses this to decide whether to mkdir or not.
-        d = self._get_node_and_metadata_for_path(path)
-        d.addErrback(self._convert_error)
-        d.addCallback(lambda res: None)
-        return d
-
-    def _convert_error(self, f):
-        if f.check(NoSuchChildError):
-            childname = f.value.args[0].encode("utf-8")
-            msg = "'%s' doesn't exist" % childname
-            raise ftp.FileNotFoundError(msg)
-        if f.check(ExistingChildError):
-            msg = f.value.args[0].encode("utf-8")
-            raise ftp.FileExistsError(msg)
-        return f
-
-    def _get_root(self, path):
-        # return (root, remaining_path)
-        path = [unicode(p) for p in path]
-        if path and path[0] == "uri":
-            d = defer.maybeDeferred(self.client.create_node_from_uri,
-                                    str(path[1]))
-            d.addCallback(lambda root: (root, path[2:]))
-        else:
-            d = defer.succeed((self.root,path))
-        return d
-
-    def _get_node_and_metadata_for_path(self, path):
-        d = self._get_root(path)
-        def _got_root(root_and_path):
-            (root,path) = root_and_path
-            if path:
-                return root.get_child_and_metadata_at_path(path)
-            else:
-                return (root,{})
-        d.addCallback(_got_root)
-        return d
-
-    def _populate_row(self, keys, childnode_and_metadata):
-        (childnode, metadata) = childnode_and_metadata
-        values = []
-        isdir = bool(IDirectoryNode.providedBy(childnode))
-        for key in keys:
-            if key == "size":
-                if isdir:
-                    value = 0
-                else:
-                    value = childnode.get_size() or 0
-            elif key == "directory":
-                value = isdir
-            elif key == "permissions":
-                # Twisted-14.0.2 (and earlier) expected an int, and used it
-                # in a rendering function that did (mode & NUMBER).
-                # Twisted-15.0.0 expects a
-                # twisted.python.filepath.Permissions , and calls its
-                # .shorthand() method. This provides both.
-                value = IntishPermissions(0o600)
-            elif key == "hardlinks":
-                value = 1
-            elif key == "modified":
-                # follow sftpd convention (i.e. linkmotime in preference to mtime)
-                if "linkmotime" in metadata.get("tahoe", {}):
-                    value = metadata["tahoe"]["linkmotime"]
-                else:
-                    value = metadata.get("mtime", 0)
-            elif key == "owner":
-                value = self.username
-            elif key == "group":
-                value = self.username
-            else:
-                value = "??"
-            values.append(value)
-        return values
-
-    def stat(self, path, keys=()):
-        # for files only, I think
-        d = self._get_node_and_metadata_for_path(path)
-        def _render(node_and_metadata):
-            (node, metadata) = node_and_metadata
-            assert not IDirectoryNode.providedBy(node)
-            return self._populate_row(keys, (node,metadata))
-        d.addCallback(_render)
-        d.addErrback(self._convert_error)
-        return d
-
-    def list(self, path, keys=()):
-        # the interface claims that path is a list of unicodes, but in
-        # practice it is not
-        d = self._get_node_and_metadata_for_path(path)
-        def _list(node_and_metadata):
-            (node, metadata) = node_and_metadata
-            if IDirectoryNode.providedBy(node):
-                return node.list()
-            return { path[-1]: (node, metadata) } # need last-edge metadata
-        d.addCallback(_list)
-        def _render(children):
-            results = []
-            for (name, childnode) in children.iteritems():
-                # the interface claims that the result should have a unicode
-                # object as the name, but it fails unless you give it a
-                # bytestring
-                results.append( (name.encode("utf-8"),
-                                 self._populate_row(keys, childnode) ) )
-            return results
-        d.addCallback(_render)
-        d.addErrback(self._convert_error)
-        return d
-
-    def openForReading(self, path):
-        d = self._get_node_and_metadata_for_path(path)
-        d.addCallback(lambda node_and_metadata: ReadFile(node_and_metadata[0]))
-        d.addErrback(self._convert_error)
-        return d
-
-    def openForWriting(self, path):
-        path = [unicode(p) for p in path]
-        if not path:
-            raise ftp.PermissionDeniedError("cannot STOR to root directory")
-        childname = path[-1]
-        d = self._get_root(path)
-        def _got_root(root_and_path):
-            (root, path) = root_and_path
-            if not path:
-                raise ftp.PermissionDeniedError("cannot STOR to root directory")
-            return root.get_child_at_path(path[:-1])
-        d.addCallback(_got_root)
-        def _got_parent(parent):
-            return WriteFile(parent, childname, self.convergence)
-        d.addCallback(_got_parent)
-        return d
-
-from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme
-
-
-@implementer(portal.IRealm)
-class Dispatcher(object):
-    def __init__(self, client):
-        self.client = client
-
-    def requestAvatar(self, avatarID, mind, interface):
-        assert interface == ftp.IFTPShell
-        rootnode = self.client.create_node_from_uri(avatarID.rootcap)
-        convergence = self.client.convergence
-        s = Handler(self.client, rootnode, avatarID.username, convergence)
-        def logout(): pass
-        return (interface, s, None)
-
-
-class FTPServer(service.MultiService):
-    def __init__(self, client, accountfile, accounturl, ftp_portstr):
-        precondition(isinstance(accountfile, (unicode, NoneType)), accountfile)
-        service.MultiService.__init__(self)
-
-        r = Dispatcher(client)
-        p = portal.Portal(r)
-
-        if accountfile:
-            c = AccountFileChecker(self, accountfile)
-            p.registerChecker(c)
-        if accounturl:
-            c = AccountURLChecker(self, accounturl)
-            p.registerChecker(c)
-        if not accountfile and not accounturl:
-            # we could leave this anonymous, with just the /uri/CAP form
-            raise NeedRootcapLookupScheme("must provide some translation")
-
-        f = ftp.FTPFactory(p)
-        # strports requires a native string.
-        ftp_portstr = ensure_str(ftp_portstr)
-        s = strports.service(ftp_portstr, f)
-        s.setServiceParent(self)
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@ -1,6 +1,17 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import six
-import heapq, traceback, array, stat, struct
-from types import NoneType
+import heapq, traceback, stat, struct
 from stat import S_IFREG, S_IFDIR
 from time import time, strftime, localtime

@ -45,6 +56,17 @@ from allmydata.util.log import NOISY, OPERATIONAL, WEIRD, \
 if six.PY3:
    long = int

+
+def createSFTPError(errorCode, errorMessage):
+    """
+    SFTPError that can accept both Unicode and bytes.
+
+    Twisted expects _native_ strings for the SFTPError message, but we often do
+    Unicode by default even on Python 2.
+    """
+    return SFTPError(errorCode, six.ensure_str(errorMessage))
+
+
 def eventually_callback(d):
    return lambda res: eventually(d.callback, res)

@ -53,9 +75,9 @@ def eventually_errback(d):


 def _utf8(x):
-    if isinstance(x, unicode):
-        return x.encode('utf-8')
    if isinstance(x, str):
+        return x.encode('utf-8')
+    if isinstance(x, bytes):
        return x
    return repr(x)

@ -64,7 +86,7 @@ def _to_sftp_time(t):
    """SFTP times are unsigned 32-bit integers representing UTC seconds
    (ignoring leap seconds) since the Unix epoch, January 1 1970 00:00 UTC.
    A Tahoe time is the corresponding float."""
-    return long(t) & long(0xFFFFFFFF)
+    return int(t) & int(0xFFFFFFFF)


 def _convert_error(res, request):
@ -73,7 +95,7 @@ def _convert_error(res, request):

    if not isinstance(res, Failure):
        logged_res = res
-        if isinstance(res, str): logged_res = "<data of length %r>" % (len(res),)
+        if isinstance(res, (bytes, str)): logged_res = "<data of length %r>" % (len(res),)
        logmsg("SUCCESS %r %r" % (request, logged_res,), level=OPERATIONAL)
        return res

@ -92,10 +114,10 @@ def _convert_error(res, request):
        raise err
    if err.check(NoSuchChildError):
        childname = _utf8(err.value.args[0])
-        raise SFTPError(FX_NO_SUCH_FILE, childname)
+        raise createSFTPError(FX_NO_SUCH_FILE, childname)
    if err.check(NotWriteableError) or err.check(ChildOfWrongTypeError):
        msg = _utf8(err.value.args[0])
-        raise SFTPError(FX_PERMISSION_DENIED, msg)
+        raise createSFTPError(FX_PERMISSION_DENIED, msg)
    if err.check(ExistingChildError):
        # Versions of SFTP after v3 (which is what twisted.conch implements)
        # define a specific error code for this case: FX_FILE_ALREADY_EXISTS.
@ -104,16 +126,16 @@ def _convert_error(res, request):
        # to translate the error to the equivalent of POSIX EEXIST, which is
        # necessary for some picky programs (such as gedit).
        msg = _utf8(err.value.args[0])
-        raise SFTPError(FX_FAILURE, msg)
+        raise createSFTPError(FX_FAILURE, msg)
    if err.check(NotImplementedError):
-        raise SFTPError(FX_OP_UNSUPPORTED, _utf8(err.value))
+        raise createSFTPError(FX_OP_UNSUPPORTED, _utf8(err.value))
    if err.check(EOFError):
-        raise SFTPError(FX_EOF, "end of file reached")
+        raise createSFTPError(FX_EOF, "end of file reached")
    if err.check(defer.FirstError):
        _convert_error(err.value.subFailure, request)

    # We assume that the error message is not anonymity-sensitive.
-    raise SFTPError(FX_FAILURE, _utf8(err.value))
+    raise createSFTPError(FX_FAILURE, _utf8(err.value))


 def _repr_flags(flags):
@ -146,7 +168,7 @@ def _lsLine(name, attrs):
    # Since we now depend on Twisted v10.1, consider calling Twisted's version.

    mode = st_mode
-    perms = array.array('c', '-'*10)
+    perms = ["-"] * 10
    ft = stat.S_IFMT(mode)
    if   stat.S_ISDIR(ft): perms[0] = 'd'
    elif stat.S_ISREG(ft): perms[0] = '-'
@ -165,7 +187,7 @@ def _lsLine(name, attrs):
    if mode&stat.S_IXOTH: perms[9] = 'x'
    # suid/sgid never set

-    l = perms.tostring()
+    l = "".join(perms)
    l += str(st_nlink).rjust(5) + ' '
    un = str(st_uid)
    l += un.ljust(9)
@ -182,6 +204,7 @@ def _lsLine(name, attrs):
        l += strftime("%b %d  %Y ", localtime(st_mtime))
    else:
        l += strftime("%b %d %H:%M ", localtime(st_mtime))
+    l = l.encode("utf-8")
    l += name
    return l

@ -223,7 +246,7 @@ def _populate_attrs(childnode, metadata, size=None):
        if childnode and size is None:
            size = childnode.get_size()
        if size is not None:
-            _assert(isinstance(size, (int, long)) and not isinstance(size, bool), size=size)
+            _assert(isinstance(size, int) and not isinstance(size, bool), size=size)
            attrs['size'] = size
        perms = S_IFREG | 0o666

@ -255,7 +278,7 @@ def _attrs_to_metadata(attrs):

    for key in attrs:
        if key == "mtime" or key == "ctime" or key == "createtime":
-            metadata[key] = long(attrs[key])
+            metadata[key] = int(attrs[key])
        elif key.startswith("ext_"):
            metadata[key] = str(attrs[key])

@ -267,7 +290,7 @@ def _attrs_to_metadata(attrs):


 def _direntry_for(filenode_or_parent, childname, filenode=None):
-    precondition(isinstance(childname, (unicode, NoneType)), childname=childname)
+    precondition(isinstance(childname, (str, type(None))), childname=childname)

    if childname is None:
        filenode_or_parent = filenode
@ -275,7 +298,7 @@ def _direntry_for(filenode_or_parent, childname, filenode=None):
    if filenode_or_parent:
        rw_uri = filenode_or_parent.get_write_uri()
        if rw_uri and childname:
-            return rw_uri + "/" + childname.encode('utf-8')
+            return rw_uri + b"/" + childname.encode('utf-8')
        else:
            return rw_uri

@ -327,7 +350,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
        if size < self.current_size or size < self.downloaded:
            self.f.truncate(size)
        if size > self.current_size:
-            self.overwrite(self.current_size, "\x00" * (size - self.current_size))
+            self.overwrite(self.current_size, b"\x00" * (size - self.current_size))
        self.current_size = size

        # make the invariant self.download_size <= self.current_size be true again
@ -335,7 +358,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
            self.download_size = size

        if self.downloaded >= self.download_size:
-            self.download_done("size changed")
+            self.download_done(b"size changed")

    def registerProducer(self, p, streaming):
        if noisy: self.log(".registerProducer(%r, streaming=%r)" % (p, streaming), level=NOISY)
@ -410,21 +433,21 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
                milestone = end

        while len(self.milestones) > 0:
-            (next, d) = self.milestones[0]
-            if next > milestone:
+            (next_, d) = self.milestones[0]
+            if next_ > milestone:
                return
-            if noisy: self.log("MILESTONE %r %r" % (next, d), level=NOISY)
+            if noisy: self.log("MILESTONE %r %r" % (next_, d), level=NOISY)
            heapq.heappop(self.milestones)
-            eventually_callback(d)("reached")
+            eventually_callback(d)(b"reached")

        if milestone >= self.download_size:
-            self.download_done("reached download size")
+            self.download_done(b"reached download size")

    def overwrite(self, offset, data):
        if noisy: self.log(".overwrite(%r, <data of length %r>)" % (offset, len(data)), level=NOISY)
        if self.is_closed:
            self.log("overwrite called on a closed OverwriteableFileConsumer", level=WEIRD)
-            raise SFTPError(FX_BAD_MESSAGE, "cannot write to a closed file handle")
+            raise createSFTPError(FX_BAD_MESSAGE, "cannot write to a closed file handle")

        if offset > self.current_size:
            # Normally writing at an offset beyond the current end-of-file
@ -435,7 +458,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
            # the gap between the current EOF and the offset.

            self.f.seek(self.current_size)
-            self.f.write("\x00" * (offset - self.current_size))
+            self.f.write(b"\x00" * (offset - self.current_size))
            start = self.current_size
        else:
            self.f.seek(offset)
@ -455,7 +478,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
        if noisy: self.log(".read(%r, %r), current_size = %r" % (offset, length, self.current_size), level=NOISY)
        if self.is_closed:
            self.log("read called on a closed OverwriteableFileConsumer", level=WEIRD)
-            raise SFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")
+            raise createSFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")

        # Note that the overwrite method is synchronous. When a write request is processed
        # (e.g. a writeChunk request on the async queue of GeneralSFTPFile), overwrite will
@ -509,7 +532,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
        return d

    def download_done(self, res):
-        _assert(isinstance(res, (str, Failure)), res=res)
+        _assert(isinstance(res, (bytes, Failure)), res=res)
        # Only the first call to download_done counts, but we log subsequent calls
        # (multiple calls are normal).
        if self.done_status is not None:
@ -526,8 +549,8 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
        eventually_callback(self.done)(None)

        while len(self.milestones) > 0:
-            (next, d) = self.milestones[0]
-            if noisy: self.log("MILESTONE FINISH %r %r %r" % (next, d, res), level=NOISY)
+            (next_, d) = self.milestones[0]
+            if noisy: self.log("MILESTONE FINISH %r %r %r" % (next_, d, res), level=NOISY)
            heapq.heappop(self.milestones)
            # The callback means that the milestone has been reached if
            # it is ever going to be. Note that the file may have been
@ -541,7 +564,7 @@ class OverwriteableFileConsumer(PrefixingLogMixin):
                self.f.close()
            except Exception as e:
                self.log("suppressed %r from close of temporary file %r" % (e, self.f), level=WEIRD)
-        self.download_done("closed")
+        self.download_done(b"closed")
        return self.done_status

    def unregisterProducer(self):
@ -565,7 +588,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):
        PrefixingLogMixin.__init__(self, facility="tahoe.sftp", prefix=userpath)
        if noisy: self.log(".__init__(%r, %r, %r)" % (userpath, filenode, metadata), level=NOISY)

-        precondition(isinstance(userpath, str) and IFileNode.providedBy(filenode),
+        precondition(isinstance(userpath, bytes) and IFileNode.providedBy(filenode),
                     userpath=userpath, filenode=filenode)
        self.filenode = filenode
        self.metadata = metadata
@ -577,7 +600,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):
        self.log(request, level=OPERATIONAL)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")
            return defer.execute(_closed)

        d = defer.Deferred()
@ -594,7 +617,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):
            # i.e. we respond with an EOF error iff offset is already at EOF.

            if offset >= len(data):
-                eventually_errback(d)(Failure(SFTPError(FX_EOF, "read at or past end of file")))
+                eventually_errback(d)(Failure(createSFTPError(FX_EOF, "read at or past end of file")))
            else:
                eventually_callback(d)(data[offset:offset+length])  # truncated if offset+length > len(data)
            return data
@ -605,7 +628,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):
    def writeChunk(self, offset, data):
        self.log(".writeChunk(%r, <data of length %r>) denied" % (offset, len(data)), level=OPERATIONAL)

-        def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
+        def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
        return defer.execute(_denied)

    def close(self):
@ -619,7 +642,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):
        self.log(request, level=OPERATIONAL)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle")
            return defer.execute(_closed)

        d = defer.execute(_populate_attrs, self.filenode, self.metadata)
@ -628,7 +651,7 @@ class ShortReadOnlySFTPFile(PrefixingLogMixin):

    def setAttrs(self, attrs):
        self.log(".setAttrs(%r) denied" % (attrs,), level=OPERATIONAL)
-        def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
+        def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
        return defer.execute(_denied)


@ -649,7 +672,7 @@ class GeneralSFTPFile(PrefixingLogMixin):
        if noisy: self.log(".__init__(%r, %r = %r, %r, <convergence censored>)" %
                           (userpath, flags, _repr_flags(flags), close_notify), level=NOISY)

-        precondition(isinstance(userpath, str), userpath=userpath)
+        precondition(isinstance(userpath, bytes), userpath=userpath)
        self.userpath = userpath
        self.flags = flags
        self.close_notify = close_notify
@ -668,11 +691,11 @@ class GeneralSFTPFile(PrefixingLogMixin):
        # not be set before then.
        self.consumer = None

-    def open(self, parent=None, childname=None, filenode=None, metadata=None):
+    def open(self, parent=None, childname=None, filenode=None, metadata=None):  # noqa: F811
        self.log(".open(parent=%r, childname=%r, filenode=%r, metadata=%r)" %
                 (parent, childname, filenode, metadata), level=OPERATIONAL)

-        precondition(isinstance(childname, (unicode, NoneType)), childname=childname)
+        precondition(isinstance(childname, (str, type(None))), childname=childname)
        precondition(filenode is None or IFileNode.providedBy(filenode), filenode=filenode)
        precondition(not self.closed, sftpfile=self)

@ -689,7 +712,7 @@ class GeneralSFTPFile(PrefixingLogMixin):
        if (self.flags & FXF_TRUNC) or not filenode:
            # We're either truncating or creating the file, so we don't need the old contents.
            self.consumer = OverwriteableFileConsumer(0, tempfile_maker)
-            self.consumer.download_done("download not needed")
+            self.consumer.download_done(b"download not needed")
        else:
            self.async_.addCallback(lambda ignored: filenode.get_best_readable_version())

@ -703,7 +726,7 @@ class GeneralSFTPFile(PrefixingLogMixin):
                d = version.read(self.consumer, 0, None)
                def _finished(res):
                    if not isinstance(res, Failure):
-                        res = "download finished"
+                        res = b"download finished"
                    self.consumer.download_done(res)
                d.addBoth(_finished)
                # It is correct to drop d here.
@ -723,7 +746,7 @@ class GeneralSFTPFile(PrefixingLogMixin):
    def rename(self, new_userpath, new_parent, new_childname):
        self.log(".rename(%r, %r, %r)" % (new_userpath, new_parent, new_childname), level=OPERATIONAL)

-        precondition(isinstance(new_userpath, str) and isinstance(new_childname, unicode),
+        precondition(isinstance(new_userpath, bytes) and isinstance(new_childname, str),
                     new_userpath=new_userpath, new_childname=new_childname)
        self.userpath = new_userpath
        self.parent = new_parent
@ -751,11 +774,11 @@ class GeneralSFTPFile(PrefixingLogMixin):
        self.log(request, level=OPERATIONAL)

        if not (self.flags & FXF_READ):
-            def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for reading")
+            def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "file handle was not opened for reading")
            return defer.execute(_denied)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot read from a closed file handle")
            return defer.execute(_closed)

        d = defer.Deferred()
@ -773,11 +796,11 @@ class GeneralSFTPFile(PrefixingLogMixin):
        self.log(".writeChunk(%r, <data of length %r>)" % (offset, len(data)), level=OPERATIONAL)

        if not (self.flags & FXF_WRITE):
-            def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
+            def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
            return defer.execute(_denied)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot write to a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot write to a closed file handle")
            return defer.execute(_closed)

        self.has_changed = True
@ -893,7 +916,7 @@ class GeneralSFTPFile(PrefixingLogMixin):
        self.log(request, level=OPERATIONAL)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot get attributes for a closed file handle")
            return defer.execute(_closed)

        # Optimization for read-only handles, when we already know the metadata.
@ -917,16 +940,16 @@ class GeneralSFTPFile(PrefixingLogMixin):
        self.log(request, level=OPERATIONAL)

        if not (self.flags & FXF_WRITE):
-            def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
+            def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "file handle was not opened for writing")
            return defer.execute(_denied)

        if self.closed:
-            def _closed(): raise SFTPError(FX_BAD_MESSAGE, "cannot set attributes for a closed file handle")
+            def _closed(): raise createSFTPError(FX_BAD_MESSAGE, "cannot set attributes for a closed file handle")
            return defer.execute(_closed)

        size = attrs.get("size", None)
-        if size is not None and (not isinstance(size, (int, long)) or size < 0):
-            def _bad(): raise SFTPError(FX_BAD_MESSAGE, "new size is not a valid nonnegative integer")
+        if size is not None and (not isinstance(size, int) or size < 0):
+            def _bad(): raise createSFTPError(FX_BAD_MESSAGE, "new size is not a valid nonnegative integer")
            return defer.execute(_bad)

        d = defer.Deferred()
@ -1012,7 +1035,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
    def logout(self):
        self.log(".logout()", level=OPERATIONAL)

-        for files in self._heisenfiles.itervalues():
+        for files in self._heisenfiles.values():
            for f in files:
                f.abandon()

@ -1039,7 +1062,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        request = "._abandon_any_heisenfiles(%r, %r)" % (userpath, direntry)
        self.log(request, level=OPERATIONAL)

-        precondition(isinstance(userpath, str), userpath=userpath)
+        precondition(isinstance(userpath, bytes), userpath=userpath)

        # First we synchronously mark all heisenfiles matching the userpath or direntry
        # as abandoned, and remove them from the two heisenfile dicts. Then we .sync()
@ -1088,8 +1111,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                   (from_userpath, from_parent, from_childname, to_userpath, to_parent, to_childname, overwrite))
        self.log(request, level=OPERATIONAL)

-        precondition((isinstance(from_userpath, str) and isinstance(from_childname, unicode) and
-                      isinstance(to_userpath, str) and isinstance(to_childname, unicode)),
+        precondition((isinstance(from_userpath, bytes) and isinstance(from_childname, str) and
+                      isinstance(to_userpath, bytes) and isinstance(to_childname, str)),
                     from_userpath=from_userpath, from_childname=from_childname, to_userpath=to_userpath, to_childname=to_childname)

        if noisy: self.log("all_heisenfiles = %r\nself._heisenfiles = %r" % (all_heisenfiles, self._heisenfiles), level=NOISY)
@ -1118,7 +1141,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        # does not mean that they were not committed; it is used to determine whether
        # a NoSuchChildError from the rename attempt should be suppressed). If overwrite
        # is False and there were already heisenfiles at the destination userpath or
-        # direntry, we return a Deferred that fails with SFTPError(FX_PERMISSION_DENIED).
+        # direntry, we return a Deferred that fails with createSFTPError(FX_PERMISSION_DENIED).

        from_direntry = _direntry_for(from_parent, from_childname)
        to_direntry = _direntry_for(to_parent, to_childname)
@ -1127,7 +1150,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                           (from_direntry, to_direntry, len(all_heisenfiles), len(self._heisenfiles), request), level=NOISY)

        if not overwrite and (to_userpath in self._heisenfiles or to_direntry in all_heisenfiles):
-            def _existing(): raise SFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + to_userpath)
+            def _existing(): raise createSFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + str(to_userpath, "utf-8"))
            if noisy: self.log("existing", level=NOISY)
            return defer.execute(_existing)

@ -1161,7 +1184,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        request = "._update_attrs_for_heisenfiles(%r, %r, %r)" % (userpath, direntry, attrs)
        self.log(request, level=OPERATIONAL)

-        _assert(isinstance(userpath, str) and isinstance(direntry, str),
+        _assert(isinstance(userpath, bytes) and isinstance(direntry, bytes),
                userpath=userpath, direntry=direntry)

        files = []
@ -1194,7 +1217,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        request = "._sync_heisenfiles(%r, %r, ignore=%r)" % (userpath, direntry, ignore)
        self.log(request, level=OPERATIONAL)

-        _assert(isinstance(userpath, str) and isinstance(direntry, (str, NoneType)),
+        _assert(isinstance(userpath, bytes) and isinstance(direntry, (bytes, type(None))),
                userpath=userpath, direntry=direntry)

        files = []
@ -1219,7 +1242,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
    def _remove_heisenfile(self, userpath, parent, childname, file_to_remove):
        if noisy: self.log("._remove_heisenfile(%r, %r, %r, %r)" % (userpath, parent, childname, file_to_remove), level=NOISY)

-        _assert(isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)),
+        _assert(isinstance(userpath, bytes) and isinstance(childname, (str, type(None))),
                userpath=userpath, childname=childname)

        direntry = _direntry_for(parent, childname)
@ -1246,8 +1269,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                           (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata),
                           level=NOISY)

-        _assert((isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)) and
-                (metadata is None or 'no-write' in metadata)),
+        _assert((isinstance(userpath, bytes) and isinstance(childname, (str, type(None))) and
+                 (metadata is None or 'no-write' in metadata)),
                userpath=userpath, childname=childname, metadata=metadata)

        writing = (flags & (FXF_WRITE | FXF_CREAT)) != 0
@ -1280,17 +1303,17 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):

        if not (flags & (FXF_READ | FXF_WRITE)):
            def _bad_readwrite():
-                raise SFTPError(FX_BAD_MESSAGE, "invalid file open flags: at least one of FXF_READ and FXF_WRITE must be set")
+                raise createSFTPError(FX_BAD_MESSAGE, "invalid file open flags: at least one of FXF_READ and FXF_WRITE must be set")
            return defer.execute(_bad_readwrite)

        if (flags & FXF_EXCL) and not (flags & FXF_CREAT):
            def _bad_exclcreat():
-                raise SFTPError(FX_BAD_MESSAGE, "invalid file open flags: FXF_EXCL cannot be set without FXF_CREAT")
+                raise createSFTPError(FX_BAD_MESSAGE, "invalid file open flags: FXF_EXCL cannot be set without FXF_CREAT")
            return defer.execute(_bad_exclcreat)

        path = self._path_from_string(pathstring)
        if not path:
-            def _emptypath(): raise SFTPError(FX_NO_SUCH_FILE, "path cannot be empty")
+            def _emptypath(): raise createSFTPError(FX_NO_SUCH_FILE, "path cannot be empty")
            return defer.execute(_emptypath)

        # The combination of flags is potentially valid.
@ -1349,20 +1372,20 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        def _got_root(root_and_path):
            (root, path) = root_and_path
            if root.is_unknown():
-                raise SFTPError(FX_PERMISSION_DENIED,
+                raise createSFTPError(FX_PERMISSION_DENIED,
                                "cannot open an unknown cap (or child of an unknown object). "
                                "Upgrading the gateway to a later Tahoe-LAFS version may help")
            if not path:
                # case 1
                if noisy: self.log("case 1: root = %r, path[:-1] = %r" % (root, path[:-1]), level=NOISY)
                if not IFileNode.providedBy(root):
-                    raise SFTPError(FX_PERMISSION_DENIED,
+                    raise createSFTPError(FX_PERMISSION_DENIED,
                                    "cannot open a directory cap")
                if (flags & FXF_WRITE) and root.is_readonly():
-                    raise SFTPError(FX_PERMISSION_DENIED,
+                    raise createSFTPError(FX_PERMISSION_DENIED,
                                    "cannot write to a non-writeable filecap without a parent directory")
                if flags & FXF_EXCL:
-                    raise SFTPError(FX_FAILURE,
+                    raise createSFTPError(FX_FAILURE,
                                    "cannot create a file exclusively when it already exists")

                # The file does not need to be added to all_heisenfiles, because it is not
@ -1389,7 +1412,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                def _got_parent(parent):
                    if noisy: self.log("_got_parent(%r)" % (parent,), level=NOISY)
                    if parent.is_unknown():
-                        raise SFTPError(FX_PERMISSION_DENIED,
+                        raise createSFTPError(FX_PERMISSION_DENIED,
                                        "cannot open a child of an unknown object. "
                                        "Upgrading the gateway to a later Tahoe-LAFS version may help")

@ -1404,13 +1427,13 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                        # which is consistent with what might happen on a POSIX filesystem.

                        if parent_readonly:
-                            raise SFTPError(FX_FAILURE,
+                            raise createSFTPError(FX_FAILURE,
                                            "cannot create a file exclusively when the parent directory is read-only")

                        # 'overwrite=False' ensures failure if the link already exists.
                        # FIXME: should use a single call to set_uri and return (child, metadata) (#1035)

-                        zero_length_lit = "URI:LIT:"
+                        zero_length_lit = b"URI:LIT:"
                        if noisy: self.log("%r.set_uri(%r, None, readcap=%r, overwrite=False)" %
                                           (parent, zero_length_lit, childname), level=NOISY)
                        d3.addCallback(lambda ign: parent.set_uri(childname, None, readcap=zero_length_lit,
@ -1436,14 +1459,14 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                        metadata['no-write'] = _no_write(parent_readonly, filenode, current_metadata)

                        if filenode.is_unknown():
-                            raise SFTPError(FX_PERMISSION_DENIED,
+                            raise createSFTPError(FX_PERMISSION_DENIED,
                                            "cannot open an unknown cap. Upgrading the gateway "
                                            "to a later Tahoe-LAFS version may help")
                        if not IFileNode.providedBy(filenode):
-                            raise SFTPError(FX_PERMISSION_DENIED,
+                            raise createSFTPError(FX_PERMISSION_DENIED,
                                            "cannot open a directory as if it were a file")
                        if (flags & FXF_WRITE) and metadata['no-write']:
-                            raise SFTPError(FX_PERMISSION_DENIED,
+                            raise createSFTPError(FX_PERMISSION_DENIED,
                                            "cannot open a non-writeable file for writing")

                        return self._make_file(file, userpath, flags, parent=parent, childname=childname,
@ -1453,10 +1476,10 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                        f.trap(NoSuchChildError)

                        if not (flags & FXF_CREAT):
-                            raise SFTPError(FX_NO_SUCH_FILE,
+                            raise createSFTPError(FX_NO_SUCH_FILE,
                                            "the file does not exist, and was not opened with the creation (CREAT) flag")
                        if parent_readonly:
-                            raise SFTPError(FX_PERMISSION_DENIED,
+                            raise createSFTPError(FX_PERMISSION_DENIED,
                                            "cannot create a file when the parent directory is read-only")

                        return self._make_file(file, userpath, flags, parent=parent, childname=childname)
@ -1495,9 +1518,9 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
            (to_parent, to_childname) = to_pair

            if from_childname is None:
-                raise SFTPError(FX_NO_SUCH_FILE, "cannot rename a source object specified by URI")
+                raise createSFTPError(FX_NO_SUCH_FILE, "cannot rename a source object specified by URI")
            if to_childname is None:
-                raise SFTPError(FX_NO_SUCH_FILE, "cannot rename to a destination specified by URI")
+                raise createSFTPError(FX_NO_SUCH_FILE, "cannot rename to a destination specified by URI")

            # <http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-6.5>
            # "It is an error if there already exists a file with the name specified
@ -1512,7 +1535,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                d2.addCallback(lambda ign: to_parent.get(to_childname))
                def _expect_fail(res):
                    if not isinstance(res, Failure):
-                        raise SFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + to_userpath)
+                        raise createSFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + str(to_userpath, "utf-8"))

                    # It is OK if we fail for errors other than NoSuchChildError, since that probably
                    # indicates some problem accessing the destination directory.
@ -1537,7 +1560,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                    if not isinstance(err, Failure) or (renamed and err.check(NoSuchChildError)):
                        return None
                    if not overwrite and err.check(ExistingChildError):
-                        raise SFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + to_userpath)
+                        raise createSFTPError(FX_PERMISSION_DENIED, "cannot rename to existing path " + str(to_userpath, "utf-8"))

                    return err
                d3.addBoth(_check)
@ -1555,7 +1578,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        path = self._path_from_string(pathstring)
        metadata = _attrs_to_metadata(attrs)
        if 'no-write' in metadata:
-            def _denied(): raise SFTPError(FX_PERMISSION_DENIED, "cannot create a directory that is initially read-only")
+            def _denied(): raise createSFTPError(FX_PERMISSION_DENIED, "cannot create a directory that is initially read-only")
            return defer.execute(_denied)

        d = self._get_root(path)
@ -1567,7 +1590,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
    def _get_or_create_directories(self, node, path, metadata):
        if not IDirectoryNode.providedBy(node):
            # TODO: provide the name of the blocking file in the error message.
-            def _blocked(): raise SFTPError(FX_FAILURE, "cannot create directory because there "
+            def _blocked(): raise createSFTPError(FX_FAILURE, "cannot create directory because there "
                                                        "is a file in the way") # close enough
            return defer.execute(_blocked)

@ -1605,7 +1628,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        def _got_parent(parent_and_childname):
            (parent, childname) = parent_and_childname
            if childname is None:
-                raise SFTPError(FX_NO_SUCH_FILE, "cannot remove an object specified by URI")
+                raise createSFTPError(FX_NO_SUCH_FILE, "cannot remove an object specified by URI")

            direntry = _direntry_for(parent, childname)
            d2 = defer.succeed(False)
@ -1636,18 +1659,18 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        d.addCallback(_got_parent_or_node)
        def _list(dirnode):
            if dirnode.is_unknown():
-                raise SFTPError(FX_PERMISSION_DENIED,
+                raise createSFTPError(FX_PERMISSION_DENIED,
                                "cannot list an unknown cap as a directory. Upgrading the gateway "
                                "to a later Tahoe-LAFS version may help")
            if not IDirectoryNode.providedBy(dirnode):
-                raise SFTPError(FX_PERMISSION_DENIED,
+                raise createSFTPError(FX_PERMISSION_DENIED,
                                "cannot list a file as if it were a directory")

            d2 = dirnode.list()
            def _render(children):
                parent_readonly = dirnode.is_readonly()
                results = []
-                for filename, (child, metadata) in children.iteritems():
+                for filename, (child, metadata) in list(children.items()):
                    # The file size may be cached or absent.
                    metadata['no-write'] = _no_write(parent_readonly, child, metadata)
                    attrs = _populate_attrs(child, metadata)
@ -1727,7 +1750,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        if "size" in attrs:
            # this would require us to download and re-upload the truncated/extended
            # file contents
-            def _unsupported(): raise SFTPError(FX_OP_UNSUPPORTED, "setAttrs wth size attribute unsupported")
+            def _unsupported(): raise createSFTPError(FX_OP_UNSUPPORTED, "setAttrs wth size attribute unsupported")
            return defer.execute(_unsupported)

        path = self._path_from_string(pathstring)
@ -1744,7 +1767,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                if childname is None:
                    if updated_heisenfiles:
                        return None
-                    raise SFTPError(FX_NO_SUCH_FILE, userpath)
+                    raise createSFTPError(FX_NO_SUCH_FILE, userpath)
                else:
                    desired_metadata = _attrs_to_metadata(attrs)
                    if noisy: self.log("desired_metadata = %r" % (desired_metadata,), level=NOISY)
@ -1767,7 +1790,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
    def readLink(self, pathstring):
        self.log(".readLink(%r)" % (pathstring,), level=OPERATIONAL)

-        def _unsupported(): raise SFTPError(FX_OP_UNSUPPORTED, "readLink")
+        def _unsupported(): raise createSFTPError(FX_OP_UNSUPPORTED, "readLink")
        return defer.execute(_unsupported)

    def makeLink(self, linkPathstring, targetPathstring):
@ -1776,7 +1799,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        # If this is implemented, note the reversal of arguments described in point 7 of
        # <http://www.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL?rev=1.15>.

-        def _unsupported(): raise SFTPError(FX_OP_UNSUPPORTED, "makeLink")
+        def _unsupported(): raise createSFTPError(FX_OP_UNSUPPORTED, "makeLink")
        return defer.execute(_unsupported)

    def extendedRequest(self, extensionName, extensionData):
@ -1785,8 +1808,8 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
        # We implement the three main OpenSSH SFTP extensions; see
        # <http://www.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL?rev=1.15>

-        if extensionName == 'posix-rename@openssh.com':
-            def _bad(): raise SFTPError(FX_BAD_MESSAGE, "could not parse posix-rename@openssh.com request")
+        if extensionName == b'posix-rename@openssh.com':
+            def _bad(): raise createSFTPError(FX_BAD_MESSAGE, "could not parse posix-rename@openssh.com request")

            if 4 > len(extensionData): return defer.execute(_bad)
            (fromPathLen,) = struct.unpack('>L', extensionData[0:4])
@ -1803,11 +1826,11 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
            # an error, or an FXP_EXTENDED_REPLY. But it happens to do the right thing
            # (respond with an FXP_STATUS message) if we return a Failure with code FX_OK.
            def _succeeded(ign):
-                raise SFTPError(FX_OK, "request succeeded")
+                raise createSFTPError(FX_OK, "request succeeded")
            d.addCallback(_succeeded)
            return d

-        if extensionName == 'statvfs@openssh.com' or extensionName == 'fstatvfs@openssh.com':
+        if extensionName == b'statvfs@openssh.com' or extensionName == b'fstatvfs@openssh.com':
            # f_bsize and f_frsize should be the same to avoid a bug in 'df'
            return defer.succeed(struct.pack('>11Q',
                1024,         # uint64  f_bsize     /* file system block size */
@ -1823,7 +1846,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
                65535,        # uint64  f_namemax   /* maximum filename length */
                ))

-        def _unsupported(): raise SFTPError(FX_OP_UNSUPPORTED, "unsupported %r request <data of length %r>" %
+        def _unsupported(): raise createSFTPError(FX_OP_UNSUPPORTED, "unsupported %r request <data of length %r>" %
                                                               (extensionName, len(extensionData)))
        return defer.execute(_unsupported)

@ -1838,29 +1861,29 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin):
    def _path_from_string(self, pathstring):
        if noisy: self.log("CONVERT %r" % (pathstring,), level=NOISY)

-        _assert(isinstance(pathstring, str), pathstring=pathstring)
+        _assert(isinstance(pathstring, bytes), pathstring=pathstring)

        # The home directory is the root directory.
-        pathstring = pathstring.strip("/")
-        if pathstring == "" or pathstring == ".":
+        pathstring = pathstring.strip(b"/")
+        if pathstring == b"" or pathstring == b".":
            path_utf8 = []
        else:
-            path_utf8 = pathstring.split("/")
+            path_utf8 = pathstring.split(b"/")

        # <http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02#section-6.2>
        # "Servers SHOULD interpret a path name component ".." as referring to
        #  the parent directory, and "." as referring to the current directory."
        path = []
        for p_utf8 in path_utf8:
-            if p_utf8 == "..":
+            if p_utf8 == b"..":
                # ignore excess .. components at the root
                if len(path) > 0:
                    path = path[:-1]
-            elif p_utf8 != ".":
+            elif p_utf8 != b".":
                try:
                    p = p_utf8.decode('utf-8', 'strict')
                except UnicodeError:
-                    raise SFTPError(FX_NO_SUCH_FILE, "path could not be decoded as UTF-8")
+                    raise createSFTPError(FX_NO_SUCH_FILE, "path could not be decoded as UTF-8")
                path.append(p)

        if noisy: self.log(" PATH %r" % (path,), level=NOISY)
@ -1979,9 +2002,9 @@ class SFTPServer(service.MultiService):

    def __init__(self, client, accountfile, accounturl,
                 sftp_portstr, pubkey_file, privkey_file):
-        precondition(isinstance(accountfile, (unicode, NoneType)), accountfile)
-        precondition(isinstance(pubkey_file, unicode), pubkey_file)
-        precondition(isinstance(privkey_file, unicode), privkey_file)
+        precondition(isinstance(accountfile, (str, type(None))), accountfile)
+        precondition(isinstance(pubkey_file, str), pubkey_file)
+        precondition(isinstance(privkey_file, str), privkey_file)
        service.MultiService.__init__(self)

        r = Dispatcher(client)
@ -2012,5 +2035,5 @@ class SFTPServer(service.MultiService):
        f = SSHFactory()
        f.portal = p

-        s = strports.service(sftp_portstr, f)
+        s = strports.service(six.ensure_str(sftp_portstr), f)
        s.setServiceParent(self)
--- a/src/allmydata/immutable/downloader/finder.py
+++ b/src/allmydata/immutable/downloader/finder.py
@ -9,6 +9,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_str

 import time
 now = time.time
@ -98,7 +99,7 @@ class ShareFinder(object):

    # internal methods
    def loop(self):
-        pending_s = ",".join([rt.server.get_name()
+        pending_s = ",".join([ensure_str(rt.server.get_name())
                              for rt in self.pending_requests]) # sort?
        self.log(format="ShareFinder loop: running=%(running)s"
                 " hungry=%(hungry)s, pending=%(pending)s",
--- a/src/allmydata/immutable/encode.py
+++ b/src/allmydata/immutable/encode.py
@ -255,11 +255,11 @@ class Encoder(object):
            # captures the slot, not the value
            #d.addCallback(lambda res: self.do_segment(i))
            # use this form instead:
-            d.addCallback(lambda res, i=i: self._encode_segment(i))
+            d.addCallback(lambda res, i=i: self._encode_segment(i, is_tail=False))
            d.addCallback(self._send_segment, i)
            d.addCallback(self._turn_barrier)
        last_segnum = self.num_segments - 1
-        d.addCallback(lambda res: self._encode_tail_segment(last_segnum))
+        d.addCallback(lambda res: self._encode_segment(last_segnum, is_tail=True))
        d.addCallback(self._send_segment, last_segnum)
        d.addCallback(self._turn_barrier)

@ -317,8 +317,24 @@ class Encoder(object):
            dl.append(d)
        return self._gather_responses(dl)

-    def _encode_segment(self, segnum):
-        codec = self._codec
+    def _encode_segment(self, segnum, is_tail):
+        """
+        Encode one segment of input into the configured number of shares.
+
+        :param segnum: Ostensibly, the number of the segment to encode.  In
+            reality, this parameter is ignored and the *next* segment is
+            encoded and returned.
+
+        :param bool is_tail: ``True`` if this is the last segment, ``False``
+            otherwise.
+
+        :return: A ``Deferred`` which fires with a two-tuple.  The first
+            element is a list of string-y objects representing the encoded
+            segment data for one of the shares.  The second element is a list
+            of integers giving the share numbers of the shares in the first
+            element.
+        """
+        codec = self._tail_codec if is_tail else self._codec
        start = time.time()

        # the ICodecEncoder API wants to receive a total of self.segment_size
@ -350,9 +366,11 @@ class Encoder(object):
        # footprint to 430KiB at the expense of more hash-tree overhead.

        d = self._gather_data(self.required_shares, input_piece_size,
-                              crypttext_segment_hasher)
+                              crypttext_segment_hasher, allow_short=is_tail)
        def _done_gathering(chunks):
            for c in chunks:
+                # If is_tail then a short trailing chunk will have been padded
+                # by _gather_data
                assert len(c) == input_piece_size
            self._crypttext_hashes.append(crypttext_segment_hasher.digest())
            # during this call, we hit 5*segsize memory
@ -365,31 +383,6 @@ class Encoder(object):
        d.addCallback(_done)
        return d

-    def _encode_tail_segment(self, segnum):
-
-        start = time.time()
-        codec = self._tail_codec
-        input_piece_size = codec.get_block_size()
-
-        crypttext_segment_hasher = hashutil.crypttext_segment_hasher()
-
-        d = self._gather_data(self.required_shares, input_piece_size,
-                              crypttext_segment_hasher, allow_short=True)
-        def _done_gathering(chunks):
-            for c in chunks:
-                # a short trailing chunk will have been padded by
-                # _gather_data
-                assert len(c) == input_piece_size
-            self._crypttext_hashes.append(crypttext_segment_hasher.digest())
-            return codec.encode(chunks)
-        d.addCallback(_done_gathering)
-        def _done(res):
-            elapsed = time.time() - start
-            self._times["cumulative_encoding"] += elapsed
-            return res
-        d.addCallback(_done)
-        return d
-
    def _gather_data(self, num_chunks, input_chunk_size,
                     crypttext_segment_hasher,
                     allow_short=False):
--- a/src/allmydata/immutable/literal.py
+++ b/src/allmydata/immutable/literal.py
@ -19,7 +19,7 @@ from twisted.protocols import basic
 from allmydata.interfaces import IImmutableFileNode, ICheckable
 from allmydata.uri import LiteralFileURI

-@implementer(IImmutableFileNode, ICheckable)
+
 class _ImmutableFileNodeBase(object):

    def get_write_uri(self):
@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object):
        return not self == other


+@implementer(IImmutableFileNode, ICheckable)
 class LiteralFileNode(_ImmutableFileNodeBase):

    def __init__(self, filecap):
--- a/src/allmydata/immutable/offloaded.py
+++ b/src/allmydata/immutable/offloaded.py
@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object):


@implementer(interfaces.RICHKUploadHelper)
-class CHKUploadHelper(Referenceable, upload.CHKUploader):
+class CHKUploadHelper(Referenceable, upload.CHKUploader):  # type: ignore # warner/foolscap#78
    """I am the helper-server -side counterpart to AssistedUploader. I handle
    peer selection, encoding, and share pushing. I read ciphertext from the
    remote AssistedUploader.
@ -499,10 +499,13 @@ class LocalCiphertextReader(AskUntilSuccessMixin):
        # ??. I'm not sure if it makes sense to forward the close message.
        return self.call("close")

+    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561
+    def set_upload_status(self, upload_status):
+        raise NotImplementedError


@implementer(interfaces.RIHelper, interfaces.IStatsProducer)
-class Helper(Referenceable):
+class Helper(Referenceable):  # type: ignore # warner/foolscap#78
    """
    :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which
        have been started but not finished, a mapping from storage index to the
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@ -11,20 +11,32 @@ from future.utils import PY2, native_str
 if PY2:
    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long, unicode
+from six import ensure_str
+
+try:
+    from typing import List
+except ImportError:
+    pass

 import os, time, weakref, itertools
+
+import attr
+
 from zope.interface import implementer
 from twisted.python import failure
 from twisted.internet import defer
 from twisted.application import service
-from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually
+from foolscap.api import Referenceable, Copyable, RemoteCopy

 from allmydata.crypto import aes
 from allmydata.util.hashutil import file_renewal_secret_hash, \
     file_cancel_secret_hash, bucket_renewal_secret_hash, \
     bucket_cancel_secret_hash, plaintext_hasher, \
     storage_index_hash, plaintext_segment_hasher, convergence_hasher
-from allmydata.util.deferredutil import timeout_call
+from allmydata.util.deferredutil import (
+    timeout_call,
+    until,
+)
 from allmydata import hashtree, uri
 from allmydata.storage.server import si_b2a
 from allmydata.immutable import encode
@ -385,6 +397,9 @@ class PeerSelector(object):
        )
        return self.happiness_mappings

+    def add_peers(self, peerids=None):
+        raise NotImplementedError
+

 class _QueryStatistics(object):

@ -896,13 +911,45 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
        raise UploadUnhappinessError(msg)


+@attr.s
+class _Accum(object):
+    """
+    Accumulate up to some known amount of ciphertext.
+
+    :ivar remaining: The number of bytes still expected.
+    :ivar ciphertext: The bytes accumulated so far.
+    """
+    remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int
+    ciphertext = attr.ib(default=attr.Factory(list))                # type: List[bytes]
+
+    def extend(self,
+               size,           # type: int
+               ciphertext,     # type: List[bytes]
+    ):
+        """
+        Accumulate some more ciphertext.
+
+        :param size: The amount of data the new ciphertext represents towards
+            the goal.  This may be more than the actual size of the given
+            ciphertext if the source has run out of data.
+
+        :param ciphertext: The new ciphertext to accumulate.
+        """
+        self.remaining -= size
+        self.ciphertext.extend(ciphertext)
+
+
@implementer(IEncryptedUploadable)
 class EncryptAnUploadable(object):
    """This is a wrapper that takes an IUploadable and provides
    IEncryptedUploadable."""
    CHUNKSIZE = 50*1024

-    def __init__(self, original, log_parent=None, progress=None):
+    def __init__(self, original, log_parent=None, progress=None, chunk_size=None):
+        """
+        :param chunk_size: The number of bytes to read from the uploadable at a
+            time, or None for some default.
+        """
        precondition(original.default_params_set,
                     "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,))
        self.original = IUploadable(original)
@ -916,6 +963,8 @@ class EncryptAnUploadable(object):
        self._ciphertext_bytes_read = 0
        self._status = None
        self._progress = progress
+        if chunk_size is not None:
+            self.CHUNKSIZE = chunk_size

    def set_upload_status(self, upload_status):
        self._status = IUploadStatus(upload_status)
@ -1022,47 +1071,53 @@ class EncryptAnUploadable(object):
        # and size
        d.addCallback(lambda ignored: self.get_size())
        d.addCallback(lambda ignored: self._get_encryptor())
-        # then fetch and encrypt the plaintext. The unusual structure here
-        # (passing a Deferred *into* a function) is needed to avoid
-        # overflowing the stack: Deferreds don't optimize out tail recursion.
-        # We also pass in a list, to which _read_encrypted will append
-        # ciphertext.
-        ciphertext = []
-        d2 = defer.Deferred()
-        d.addCallback(lambda ignored:
-                      self._read_encrypted(length, ciphertext, hash_only, d2))
-        d.addCallback(lambda ignored: d2)
+
+        accum = _Accum(length)
+
+        def action():
+            """
+            Read some bytes into the accumulator.
+            """
+            return self._read_encrypted(accum, hash_only)
+
+        def condition():
+            """
+            Check to see if the accumulator has all the data.
+            """
+            return accum.remaining == 0
+
+        d.addCallback(lambda ignored: until(action, condition))
+        d.addCallback(lambda ignored: accum.ciphertext)
        return d

-    def _read_encrypted(self, remaining, ciphertext, hash_only, fire_when_done):
-        if not remaining:
-            fire_when_done.callback(ciphertext)
-            return None
+    def _read_encrypted(self,
+                        ciphertext_accum,  # type: _Accum
+                        hash_only,         # type: bool
+    ):
+        # type: (...) -> defer.Deferred
+        """
+        Read the next chunk of plaintext, encrypt it, and extend the accumulator
+        with the resulting ciphertext.
+        """
        # tolerate large length= values without consuming a lot of RAM by
        # reading just a chunk (say 50kB) at a time. This only really matters
        # when hash_only==True (i.e. resuming an interrupted upload), since
        # that's the case where we will be skipping over a lot of data.
-        size = min(remaining, self.CHUNKSIZE)
-        remaining = remaining - size
+        size = min(ciphertext_accum.remaining, self.CHUNKSIZE)
+
        # read a chunk of plaintext..
        d = defer.maybeDeferred(self.original.read, size)
-        # N.B.: if read() is synchronous, then since everything else is
-        # actually synchronous too, we'd blow the stack unless we stall for a
-        # tick. Once you accept a Deferred from IUploadable.read(), you must
-        # be prepared to have it fire immediately too.
-        d.addCallback(fireEventually)
        def _good(plaintext):
            # and encrypt it..
            # o/' over the fields we go, hashing all the way, sHA! sHA! sHA! o/'
            ct = self._hash_and_encrypt_plaintext(plaintext, hash_only)
-            ciphertext.extend(ct)
-            self._read_encrypted(remaining, ciphertext, hash_only,
-                                 fire_when_done)
-        def _err(why):
-            fire_when_done.errback(why)
+            # Intentionally tell the accumulator about the expected size, not
+            # the actual size.  If we run out of data we still want remaining
+            # to drop otherwise it will never reach 0 and the loop will never
+            # end.
+            ciphertext_accum.extend(size, ct)
        d.addCallback(_good)
-        d.addErrback(_err)
-        return None
+        return d

    def _hash_and_encrypt_plaintext(self, data, hash_only):
        assert isinstance(data, (tuple, list)), type(data)
@ -1423,7 +1478,7 @@ class LiteralUploader(object):
        return self._status

@implementer(RIEncryptedUploadable)
-class RemoteEncryptedUploadable(Referenceable):
+class RemoteEncryptedUploadable(Referenceable):  # type: ignore # warner/foolscap#78

    def __init__(self, encrypted_uploadable, upload_status):
        self._eu = IEncryptedUploadable(encrypted_uploadable)
@ -1825,7 +1880,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
    def startService(self):
        service.MultiService.startService(self)
        if self._helper_furl:
-            self.parent.tub.connectTo(self._helper_furl,
+            self.parent.tub.connectTo(ensure_str(self._helper_furl),
                                      self._got_helper)

    def _got_helper(self, helper):
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@ -681,7 +681,7 @@ class IURI(Interface):
        passing into init_from_string."""


-class IVerifierURI(Interface, IURI):
+class IVerifierURI(IURI):
    def init_from_string(uri):
        """Accept a string (as created by my to_string() method) and populate
        this instance with its data. I am not normally called directly,
@ -748,7 +748,7 @@ class IProgress(Interface):
        "Current amount of progress (in percentage)"
    )

-    def set_progress(self, value):
+    def set_progress(value):
        """
        Sets the current amount of progress.

@ -756,7 +756,7 @@ class IProgress(Interface):
        set_progress_total.
        """

-    def set_progress_total(self, value):
+    def set_progress_total(value):
        """
        Sets the total amount of expected progress

@ -859,12 +859,6 @@ class IPeerSelector(Interface):
        peer selection begins.
        """

-    def confirm_share_allocation(peerid, shnum):
-        """
-        Confirm that an allocated peer=>share pairing has been
-        successfully established.
-        """
-
    def add_peers(peerids=set):
        """
        Update my internal state to include the peers in peerids as
@ -1824,11 +1818,6 @@ class IEncoder(Interface):
    willing to receive data.
    """

-    def set_size(size):
-        """Specify the number of bytes that will be encoded. This must be
-        peformed before get_serialized_params() can be called.
-        """
-
    def set_encrypted_uploadable(u):
        """Provide a source of encrypted upload data. 'u' must implement
        IEncryptedUploadable.
--- a/src/allmydata/introducer/client.py
+++ b/src/allmydata/introducer/client.py
@ -11,12 +11,12 @@ if PY2:
    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long

-from six import ensure_text
+from six import ensure_text, ensure_str

 import time
 from zope.interface import implementer
 from twisted.application import service
-from foolscap.api import Referenceable, eventually
+from foolscap.api import Referenceable
 from allmydata.interfaces import InsufficientVersionError
 from allmydata.introducer.interfaces import IIntroducerClient, \
     RIIntroducerSubscriberClient_v2
@ -24,6 +24,9 @@ from allmydata.introducer.common import sign_to_foolscap, unsign_from_foolscap,\
     get_tubid_string_from_ann
 from allmydata.util import log, yamlutil, connection_status
 from allmydata.util.rrefutil import add_version_to_remote_reference
+from allmydata.util.observer import (
+    ObserverList,
+)
 from allmydata.crypto.error import BadSignature
 from allmydata.util.assertutil import precondition

@ -39,8 +42,6 @@ class IntroducerClient(service.Service, Referenceable):
                 nickname, my_version, oldest_supported,
                 sequencer, cache_filepath):
        self._tub = tub
-        if isinstance(introducer_furl, str):
-            introducer_furl = introducer_furl.encode("utf-8")
        self.introducer_furl = introducer_furl

        assert isinstance(nickname, str)
@ -64,8 +65,7 @@ class IntroducerClient(service.Service, Referenceable):
        self._publisher = None
        self._since = None

-        self._local_subscribers = [] # (servicename,cb,args,kwargs) tuples
-        self._subscribed_service_names = set()
+        self._local_subscribers = {} # {servicename: ObserverList}
        self._subscriptions = set() # requests we've actually sent

        # _inbound_announcements remembers one announcement per
@ -96,7 +96,7 @@ class IntroducerClient(service.Service, Referenceable):
    def startService(self):
        service.Service.startService(self)
        self._introducer_error = None
-        rc = self._tub.connectTo(self.introducer_furl, self._got_introducer)
+        rc = self._tub.connectTo(ensure_str(self.introducer_furl), self._got_introducer)
        self._introducer_reconnector = rc
        def connect_failed(failure):
            self.log("Initial Introducer connection failed: perhaps it's down",
@ -178,22 +178,22 @@ class IntroducerClient(service.Service, Referenceable):
            kwargs["facility"] = "tahoe.introducer.client"
        return log.msg(*args, **kwargs)

-    def subscribe_to(self, service_name, cb, *args, **kwargs):
-        self._local_subscribers.append( (service_name,cb,args,kwargs) )
-        self._subscribed_service_names.add(service_name)
+    def subscribe_to(self, service_name, callback, *args, **kwargs):
+        obs = self._local_subscribers.setdefault(service_name, ObserverList())
+        obs.subscribe(lambda key_s, ann: callback(key_s, ann, *args, **kwargs))
        self._maybe_subscribe()
        for index,(ann,key_s,when) in list(self._inbound_announcements.items()):
            precondition(isinstance(key_s, bytes), key_s)
            servicename = index[0]
            if servicename == service_name:
-                eventually(cb, key_s, ann, *args, **kwargs)
+                obs.notify(key_s, ann)

    def _maybe_subscribe(self):
        if not self._publisher:
            self.log("want to subscribe, but no introducer yet",
                     level=log.NOISY)
            return
-        for service_name in self._subscribed_service_names:
+        for service_name in self._local_subscribers:
            if service_name in self._subscriptions:
                continue
            self._subscriptions.add(service_name)
@ -272,7 +272,7 @@ class IntroducerClient(service.Service, Referenceable):
        precondition(isinstance(key_s, bytes), key_s)
        self._debug_counts["inbound_announcement"] += 1
        service_name = str(ann["service-name"])
-        if service_name not in self._subscribed_service_names:
+        if service_name not in self._local_subscribers:
            self.log("announcement for a service we don't care about [%s]"
                     % (service_name,), level=log.UNUSUAL, umid="dIpGNA")
            self._debug_counts["wrong_service"] += 1
@ -343,9 +343,9 @@ class IntroducerClient(service.Service, Referenceable):
    def _deliver_announcements(self, key_s, ann):
        precondition(isinstance(key_s, bytes), key_s)
        service_name = str(ann["service-name"])
-        for (service_name2,cb,args,kwargs) in self._local_subscribers:
-            if service_name2 == service_name:
-                eventually(cb, key_s, ann, *args, **kwargs)
+        obs = self._local_subscribers.get(service_name)
+        if obs is not None:
+            obs.notify(key_s, ann)

    def connection_status(self):
        assert self.running # startService builds _introducer_reconnector
--- a/src/allmydata/introducer/interfaces.py
+++ b/src/allmydata/introducer/interfaces.py
@ -73,7 +73,7 @@ class IIntroducerClient(Interface):
    publish their services to the rest of the world, and I help them learn
    about services available on other nodes."""

-    def publish(service_name, ann, signing_key=None):
+    def publish(service_name, ann, signing_key):
        """Publish the given announcement dictionary (which must be
        JSON-serializable), plus some additional keys, to the world.

@ -83,8 +83,7 @@ class IIntroducerClient(Interface):
        the signing_key, if present, otherwise it is derived from the
        'anonymous-storage-FURL' key.

-        If signing_key= is set to an instance of SigningKey, it will be
-        used to sign the announcement."""
+        signing_key (a SigningKey) will be used to sign the announcement."""

    def subscribe_to(service_name, callback, *args, **kwargs):
        """Call this if you will eventually want to use services with the
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@ -15,6 +15,12 @@ from past.builtins import long
 from six import ensure_text

 import time, os.path, textwrap
+
+try:
+    from typing import Any, Dict, Union
+except ImportError:
+    pass
+
 from zope.interface import implementer
 from twisted.application import service
 from twisted.internet import defer
@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable):
    name = "introducer"
    # v1 is the original protocol, added in 1.0 (but only advertised starting
    # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
-    VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
+    # TODO: reconcile bytes/str for keys
+    VERSION = {
+                #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
                b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
                b"application-version": allmydata.__full_version__.encode("utf-8"),
-                }
+                }  # type: Dict[Union[bytes, str], Any]

    def __init__(self):
        service.MultiService.__init__(self)
--- a/src/allmydata/mutable/filenode.py
+++ b/src/allmydata/mutable/filenode.py
@ -564,7 +564,7 @@ class MutableFileNode(object):
        return d


-    def upload(self, new_contents, servermap):
+    def upload(self, new_contents, servermap, progress=None):
        """
        I overwrite the contents of the best recoverable version of this
        mutable file with new_contents, using servermap instead of
@ -951,7 +951,7 @@ class MutableFileVersion(object):
        return self._servermap.size_of_version(self._version)


-    def download_to_data(self, fetch_privkey=False, progress=None):
+    def download_to_data(self, fetch_privkey=False, progress=None):  # type: ignore # fixme
        """
        I return a Deferred that fires with the contents of this
        readable object as a byte string.
@ -1205,3 +1205,7 @@ class MutableFileVersion(object):
                                 self._servermap,
                                 mode=mode)
        return u.update()
+
+    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562
+    def get_servermap(self):
+        raise NotImplementedError
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@ -23,6 +23,11 @@ from base64 import b32decode, b32encode
 from errno import ENOENT, EPERM
 from warnings import warn

+try:
+    from typing import Union
+except ImportError:
+    pass
+
 import attr

 # On Python 2 this will be the backported package.
@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files):
        raise e


+def ensure_text_and_abspath_expanduser_unicode(basedir):
+    # type: (Union[bytes, str]) -> str
+    return abspath_expanduser_unicode(ensure_text(basedir))
+
+
@attr.s
 class _Config(object):
    """
@ -300,8 +310,8 @@ class _Config(object):
    config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser))
    portnum_fname = attr.ib()
    _basedir = attr.ib(
-        converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)),
-    )
+        converter=ensure_text_and_abspath_expanduser_unicode,
+    )  # type: str
    config_path = attr.ib(
        validator=attr.validators.optional(
            attr.validators.instance_of(FilePath),
@ -669,8 +679,8 @@ def create_connection_handlers(config, i2p_provider, tor_provider):
    # create that handler, so hints which want it will be ignored.
    handlers = {
        "tcp": _make_tcp_handler(),
-        "tor": tor_provider.get_tor_handler(),
-        "i2p": i2p_provider.get_i2p_handler(),
+        "tor": tor_provider.get_client_endpoint(),
+        "i2p": i2p_provider.get_client_endpoint(),
    }
    log.msg(
        format="built Foolscap connection handlers for: %(known_handlers)s",
@ -927,7 +937,6 @@ class Node(service.MultiService):
    """
    NODETYPE = "unknown NODETYPE"
    CERTFILE = "node.pem"
-    GENERATED_FILES = []

    def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider):
        """
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@ -1,5 +1,10 @@
 from __future__ import print_function

+try:
+    from allmydata.scripts.types_ import SubCommands
+except ImportError:
+    pass
+
 from twisted.python import usage
 from allmydata.scripts.common import BaseOptions

@ -79,8 +84,8 @@ def do_admin(options):


 subCommands = [
-    ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"],
-    ]
+    ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"),
+    ]  # type: SubCommands

 dispatch = {
    "admin": do_admin,
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@ -1,6 +1,12 @@
 from __future__ import print_function

 import os.path, re, fnmatch
+
+try:
+    from allmydata.scripts.types_ import SubCommands, Parameters
+except ImportError:
+    pass
+
 from twisted.python import usage
 from allmydata.scripts.common import get_aliases, get_default_nodedir, \
     DEFAULT_ALIAS, BaseOptions
@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions):
         "This overrides the URL found in the --node-directory ."],
        ["dir-cap", None, None,
         "Specify which dirnode URI should be used as the 'tahoe' alias."]
-        ]
+        ]  # type: Parameters

    def postOptions(self):
        self["quiet"] = self.parent["quiet"]
@ -455,25 +461,25 @@ class DeepCheckOptions(FileStoreOptions):
    Optionally repair any problems found."""

 subCommands = [
-    ["mkdir", None, MakeDirectoryOptions, "Create a new directory."],
-    ["add-alias", None, AddAliasOptions, "Add a new alias cap."],
-    ["create-alias", None, CreateAliasOptions, "Create a new alias cap."],
-    ["list-aliases", None, ListAliasesOptions, "List all alias caps."],
-    ["ls", None, ListOptions, "List a directory."],
-    ["get", None, GetOptions, "Retrieve a file from the grid."],
-    ["put", None, PutOptions, "Upload a file into the grid."],
-    ["cp", None, CpOptions, "Copy one or more files or directories."],
-    ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."],
-    ["mv", None, MvOptions, "Move a file within the grid."],
-    ["ln", None, LnOptions, "Make an additional link to an existing file or directory."],
-    ["backup", None, BackupOptions, "Make target dir look like local dir."],
-    ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."],
-    ["manifest", None, ManifestOptions, "List all files/directories in a subtree."],
-    ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."],
-    ["check", None, CheckOptions, "Check a single file or directory."],
-    ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."],
-    ["status", None, TahoeStatusCommand, "Various status information."],
-    ]
+    ("mkdir", None, MakeDirectoryOptions, "Create a new directory."),
+    ("add-alias", None, AddAliasOptions, "Add a new alias cap."),
+    ("create-alias", None, CreateAliasOptions, "Create a new alias cap."),
+    ("list-aliases", None, ListAliasesOptions, "List all alias caps."),
+    ("ls", None, ListOptions, "List a directory."),
+    ("get", None, GetOptions, "Retrieve a file from the grid."),
+    ("put", None, PutOptions, "Upload a file into the grid."),
+    ("cp", None, CpOptions, "Copy one or more files or directories."),
+    ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."),
+    ("mv", None, MvOptions, "Move a file within the grid."),
+    ("ln", None, LnOptions, "Make an additional link to an existing file or directory."),
+    ("backup", None, BackupOptions, "Make target dir look like local dir."),
+    ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."),
+    ("manifest", None, ManifestOptions, "List all files/directories in a subtree."),
+    ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."),
+    ("check", None, CheckOptions, "Check a single file or directory."),
+    ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."),
+    ("status", None, TahoeStatusCommand, "Various status information."),
+    ]  # type: SubCommands

 def mkdir(options):
    from allmydata.scripts import tahoe_mkdir
@ -495,7 +501,7 @@ def list_aliases(options):
    rc = tahoe_add_alias.list_aliases(options)
    return rc

-def list(options):
+def list_(options):
    from allmydata.scripts import tahoe_ls
    rc = tahoe_ls.list(options)
    return rc
@ -581,7 +587,7 @@ dispatch = {
    "add-alias": add_alias,
    "create-alias": create_alias,
    "list-aliases": list_aliases,
-    "ls": list,
+    "ls": list_,
    "get": get,
    "put": put,
    "cp": cp,
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@ -4,6 +4,12 @@ import os, sys, urllib, textwrap
 import codecs
 from os.path import join

+try:
+    from typing import Optional
+    from .types_ import Parameters
+except ImportError:
+    pass
+
 from yaml import (
    safe_dump,
 )
@ -41,8 +47,8 @@ class BaseOptions(usage.Options):
    def opt_version(self):
        raise usage.UsageError("--version not allowed on subcommands")

-    description = None
-    description_unwrapped = None
+    description = None  # type: Optional[str]
+    description_unwrapped = None  # type: Optional[str]

    def __str__(self):
        width = int(os.environ.get('COLUMNS', '80'))
@ -65,7 +71,7 @@ class BasedirOptions(BaseOptions):
    optParameters = [
        ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
         % quote_local_unicode_path(_default_nodedir)],
-    ]
+    ]  # type: Parameters

    def parseArgs(self, basedir=None):
        # This finds the node-directory option correctly even if we are in a subcommand.
@ -102,7 +108,7 @@ class NoDefaultBasedirOptions(BasedirOptions):

    optParameters = [
        ["basedir", "C", None, "Specify which Tahoe base directory should be used."],
-    ]
+    ]  # type: Parameters

    # This is overridden in order to ensure we get a "Wrong number of arguments."
    # error when more than one argument is given.
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@ -3,6 +3,11 @@ from __future__ import print_function
 import os
 import json

+try:
+    from allmydata.scripts.types_ import SubCommands
+except ImportError:
+    pass
+
 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
 from twisted.python.filepath import (
@ -492,10 +497,10 @@ def create_introducer(config):


 subCommands = [
-    ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."],
-    ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."],
-    ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."],
-]
+    ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."),
+    ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."),
+    ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."),
+]  # type: SubCommands

 dispatch = {
    "create-node": create_node,
--- a/src/allmydata/scripts/debug.py
+++ b/src/allmydata/scripts/debug.py
@ -1,5 +1,12 @@
 from __future__ import print_function

+try:
+    from allmydata.scripts.types_ import SubCommands
+except ImportError:
+    pass
+
+from future.utils import bchr
+
 # do not import any allmydata modules at this level. Do that from inside
 # individual functions instead.
 import struct, time, os, sys
@ -905,7 +912,7 @@ def corrupt_share(options):
        f = open(fn, "rb+")
        f.seek(offset)
        d = f.read(1)
-        d = chr(ord(d) ^ 0x01)
+        d = bchr(ord(d) ^ 0x01)
        f.seek(offset)
        f.write(d)
        f.close()
@ -920,7 +927,7 @@ def corrupt_share(options):
        f.seek(m.DATA_OFFSET)
        data = f.read(2000)
        # make sure this slot contains an SMDF share
-        assert data[0] == b"\x00", "non-SDMF mutable shares not supported"
+        assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported"
        f.close()

        (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
@ -1051,8 +1058,8 @@ def do_debug(options):


 subCommands = [
-    ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."],
-    ]
+    ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
+    ]  # type: SubCommands

 dispatch = {
    "debug": do_debug,
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@ -4,6 +4,11 @@ import os, sys
 from six.moves import StringIO
 import six

+try:
+    from allmydata.scripts.types_ import SubCommands
+except ImportError:
+    pass
+
 from twisted.python import usage
 from twisted.internet import defer, task, threads

@ -40,8 +45,8 @@ _control_node_dispatch = {
 }

 process_control_commands = [
-    ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"],
-]
+    ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"),
+]  # type: SubCommands


 class Options(usage.Options):
@ -98,7 +103,7 @@ class Options(usage.Options):

 create_dispatch = {}
 for module in (create_node,):
-    create_dispatch.update(module.dispatch)
+    create_dispatch.update(module.dispatch)  # type: ignore

 def parse_options(argv, config=None):
    if not config:
--- a/src/allmydata/scripts/tahoe_invite.py
+++ b/src/allmydata/scripts/tahoe_invite.py
@ -2,6 +2,11 @@ from __future__ import print_function

 import json

+try:
+    from allmydata.scripts.types_ import SubCommands
+except ImportError:
+    pass
+
 from twisted.python import usage
 from twisted.internet import defer, reactor

@ -103,7 +108,7 @@ def invite(options):
 subCommands = [
    ("invite", None, InviteOptions,
     "Invite a new node to this grid"),
-]
+]  # type: SubCommands

 dispatch = {
    "invite": invite,
--- a/src/allmydata/scripts/types_.py
+++ b/src/allmydata/scripts/types_.py
@ -0,0 +1,12 @@
+from typing import List, Tuple, Type, Sequence, Any
+from allmydata.scripts.common import BaseOptions
+
+
+# Historically, subcommands were implemented as lists, but due to a
+# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170),
+# a Tuple is required.
+SubCommand = Tuple[str, None, Type[BaseOptions], str]
+
+SubCommands = List[SubCommand]
+
+Parameters = List[Sequence[Any]]
--- a/src/allmydata/stats.py
+++ b/src/allmydata/stats.py
@ -1,11 +1,16 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals

-import time
-
-# Python 2 compatibility
 from future.utils import PY2
 if PY2:
-    from future.builtins import str  # noqa: F401
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+import time

 from twisted.application import service
 from twisted.application.internet import TimerService
@ -18,7 +23,7 @@ from allmydata.interfaces import IStatsProducer
@implementer(IStatsProducer)
 class CPUUsageMonitor(service.MultiService):
    HISTORY_LENGTH = 15
-    POLL_INTERVAL = 60
+    POLL_INTERVAL = 60  # type: float

    def __init__(self):
        service.MultiService.__init__(self)
--- a/src/allmydata/storage/crawler.py
+++ b/src/allmydata/storage/crawler.py
@ -19,7 +19,7 @@ import os, time, struct
 try:
    import cPickle as pickle
 except ImportError:
-    import pickle
+    import pickle  # type: ignore
 from twisted.internet import reactor
 from twisted.application import service
 from allmydata.storage.common import si_b2a
--- a/src/allmydata/storage/immutable.py
+++ b/src/allmydata/storage/immutable.py
@ -202,7 +202,7 @@ class ShareFile(object):


@implementer(RIBucketWriter)
-class BucketWriter(Referenceable):
+class BucketWriter(Referenceable):  # type: ignore # warner/foolscap#78

    def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary):
        self.ss = ss
@ -301,7 +301,7 @@ class BucketWriter(Referenceable):


@implementer(RIBucketReader)
-class BucketReader(Referenceable):
+class BucketReader(Referenceable):  # type: ignore # warner/foolscap#78

    def __init__(self, ss, sharefname, storage_index=None, shnum=None):
        self.ss = ss
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable):
        for share in six.viewvalues(shares):
            share.add_or_renew_lease(lease_info)

-    def slot_testv_and_readv_and_writev(
+    def slot_testv_and_readv_and_writev(  # type: ignore # warner/foolscap#78
            self,
            storage_index,
            secrets,
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@ -464,6 +464,7 @@ class StorageFarmBroker(service.MultiService):
@implementer(IDisplayableServer)
 class StubServer(object):
    def __init__(self, serverid):
+        assert isinstance(serverid, bytes)
        self.serverid = serverid # binary tubid
    def get_serverid(self):
        return self.serverid
--- a/src/allmydata/test/init.py
+++ b/src/allmydata/test/init.py
@ -113,4 +113,5 @@ if sys.platform == "win32":
    initialize()

 from eliot import to_file
-to_file(open("eliot.log", "w"))
+from allmydata.util.jsonbytes import BytesJSONEncoder
+to_file(open("eliot.log", "w"), encoder=BytesJSONEncoder)
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`Debian 8 support has been replaced with Debian 10 support.`
				`@ -0,0 +1 @@`
				`Added 'typechecks' environment for tox running mypy and performing static typechecks.`
				`@ -0,0 +1 @@`
				`FTP is no longer supported by Tahoe-LAFS. Please use the SFTP support instead.`
				`@ -0,0 +1 @@`
				`SFTP public key auth likely works more consistently, and SFTP in general was previously broken.`
				`@ -0,0 +1 @@`
				`The Tahoe command line now always uses UTF-8 to decode its arguments, regardless of locale.`
				`@ -0,0 +1 @@`
				`Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work.`