Merge remote-tracking branch 'origin/master' into 3636.doc-toc-reorg

.github/workflows/ci.yml

@@ -19,7 +19,6 @@ jobs:
       matrix:
         os:
           - windows-latest
-          - macos-latest
           - ubuntu-latest
         python-version:
           - 2.7
@@ -27,6 +26,12 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
+        include:
+          # On macOS don't bother with 3.6-3.8, just to get faster builds.
+          - os: macos-latest
+            python-version: 2.7
+          - os: macos-latest
+            python-version: 3.9
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0
@@ -37,33 +42,10 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
-      # See note below about need for using 32-bit Python 2.7 on
-      # Windows.
-      - name: Set up Python ${{ matrix.python-version }} [Windows x64]
-        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version != '2.7' ) }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x64'
-
-      # We use netifaces, which does not ship a 64-bit wheel for the
-      # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
-      # Since MS has removed vcpython27 compiler downloads from their
-      # usual download site, building a netifaces wheel locally is not
-      # an option anymore.  So let us just test with 32-bit Python on
-      # Windows.
-      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '2.7' ) }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x86'
-
       # To use pip caching with GitHub Actions in an OS-independent
       # manner, we need `pip cache dir` command, which became
       # available since pip v20.1+.  At the time of writing this,
@@ -173,23 +155,34 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - macos-latest
           - windows-latest
+          - ubuntu-latest
         python-version:
           - 2.7
+          - 3.6
+          - 3.9
         include:
-          - os: ubuntu-latest
+          # On macOS don't bother with 3.6, just to get faster builds.
-            python-version: 3.6
+          - os: macos-latest
+            python-version: 2.7
+          - os: macos-latest
+            python-version: 3.9
 
     steps:
 
       - name: Install Tor [Ubuntu]
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt install tor
-
+        
-      - name: Install Tor [macOS]
+      # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744.
-        if: matrix.os == 'macos-latest'
+      # We have to use an older version of Tor for running integration
-        run: brew install tor
+      # tests on macOS.
+      - name: Install Tor [macOS, ${{ matrix.python-version }} ]
+        if: ${{ matrix.os == 'macos-latest' }}
+        run: |
+          brew extract --version 0.4.5.8 tor homebrew/cask
+          brew install tor@0.4.5.8
+          brew link --overwrite tor@0.4.5.8
 
       - name: Install Tor [Windows]
         if: matrix.os == 'windows-latest'
@@ -203,19 +196,10 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
-      # See this step under coverage job.
-      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x86'
-
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -272,19 +256,10 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
-      # See this step under coverage job.
-      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' }}
-        uses: actions/setup-python@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          architecture: 'x86'
-
       - name: Get pip cache directory
         id: pip-cache
         run: |

docs/INSTALL.rst

@@ -1,343 +0,0 @@
-.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*-
-
-..
-    note: if you aren't reading the rendered form of these docs at
-    http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any
-    ":doc:" links refer to other files in this docs/ directory
-
-*********************
-Installing Tahoe-LAFS
-*********************
-
-Welcome to `the Tahoe-LAFS project`_, a secure, decentralized, fault-tolerant
-storage system. See :doc:`about` for an overview of the architecture and
-security properties of the system.
-
-This procedure should work on Windows, Mac, illumos (previously OpenSolaris),
-and too many flavors of Linux and of BSD to list.
-
-.. _the Tahoe-LAFS project: https://tahoe-lafs.org
-
-First: In Case Of Trouble
-=========================
-
-In some cases these instructions may fail due to peculiarities of your
-platform.
-
-If the following instructions don't Just Work without any further effort on
-your part, then please write to `the tahoe-dev mailing list`_ where friendly
-hackers will help you out.
-
-.. _the tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev
-
-Pre-Packaged Versions
-=====================
-
-You may not need to build Tahoe at all.
-
-If you are on Windows, please see :doc:`windows` for platform-specific
-instructions.
-
-If you are on a Mac, you can either follow these instructions, or use the
-pre-packaged bundle described in :doc:`OS-X`.
-
-Many Linux distributions include Tahoe-LAFS packages. Debian and Ubuntu users
-can ``apt-get install tahoe-lafs``. See `OSPackages`_ for other
-platforms.
-
-.. _OSPackages: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/OSPackages
-
-
-Preliminaries
-=============
-
-If you don't use a pre-packaged copy of Tahoe, you can build it yourself.
-You'll need Python2.7, pip, and virtualenv.
-Tahoe-LAFS depends on some libraries which require a C compiler to build.
-However, for many platforms, PyPI hosts already-built packages of libraries.
-
-If there is no already-built package for your platform,
-you will need a C compiler,
-the Python development headers,
-and some libraries (libffi-dev and libssl-dev).
-
-On a modern Debian/Ubuntu-derived distribution, this command will get you
-everything you need::
-
-    apt-get install build-essential python-dev libffi-dev libssl-dev libyaml-dev python-virtualenv
-
-On OS-X, install pip and virtualenv as described below. If you want to
-compile the dependencies yourself, you'll also need to install
-Xcode and its command-line tools.
-
-**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
-
-Python 2.7
-----------
-
-Check if you already have an adequate version of Python installed by running
-``python -V``. The latest version of Python v2.7 is recommended, which is
-2.7.11 as of this writing. Python v2.6.x and v3 do not work. On Windows, we
-recommend the use of native Python v2.7, not Cygwin Python. If you don't have
-one of these versions of Python installed, `download`_ and install the latest
-version of Python v2.7. Make sure that the path to the installation directory
-has no spaces in it (e.g. on Windows, do not install Python in the "Program
-Files" directory)::
-
-    % python --version
-    Python 2.7.11
-
-.. _download: https://www.python.org/downloads/
-
-pip
----
-
-Many Python installations already include ``pip``, but in case yours does
-not, get it with the `pip install instructions`_::
-
-    % pip --version
-    pip 10.0.1 from ... (python 2.7)
-
-.. _pip install instructions: https://pip.pypa.io/en/stable/installing/
-
-virtualenv
-----------
-
-If you do not have an OS-provided copy of ``virtualenv``, install it with the
-instructions from the `virtualenv documentation`_::
-
-
-    % virtualenv --version
-    15.1.0
-
-.. _virtualenv documentation: https://virtualenv.pypa.io/en/latest/installation.html
-
-C compiler and libraries
-------------------------
-
-Except on OS-X, where the Tahoe project hosts pre-compiled wheels for all
-dependencies, you will need several C libraries installed before you can
-build. You will also need the Python development headers, and a C compiler
-(your python installation should know how to find these).
-
-On Debian/Ubuntu-derived systems, the necessary packages are ``python-dev``,
-``libffi-dev``, and ``libssl-dev``, and can be installed with ``apt-get``. On
-RPM-based system (like Fedora) these may be named ``python-devel``, etc,
-instead, and cam be installed with ``yum`` or ``rpm``.
-
-**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
-
-
-Install the Latest Tahoe-LAFS Release
-=====================================
-
-We recommend creating a fresh virtualenv for your Tahoe-LAFS install, to
-isolate it from any python packages that are already installed (and to
-isolate the rest of your system from Tahoe's dependencies).
-
-This example uses a virtualenv named ``venv``, but you can call it anything
-you like. Many people prefer to keep all their virtualenvs in one place, like
-``~/.local/venvs/`` or ``~/venvs/``.
-
-It's usually a good idea to upgrade the virtualenv's ``pip`` and
-``setuptools`` to their latest versions, with ``venv/bin/pip install -U pip
-setuptools``. Many operating systems have an older version of ``virtualenv``,
-which then includes older versions of pip and setuptools. Upgrading is easy,
-and only affects the virtualenv: not the rest of your computer.
-
-Then use the virtualenv's ``pip`` to install the latest Tahoe-LAFS release
-from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run
-``venv/bin/tahoe --version`` to confirm the install was successful::
-
- % virtualenv venv
- New python executable in ~/venv/bin/python2.7
- Installing setuptools, pip, wheel...done.
-
- % venv/bin/pip install -U pip setuptools
- Downloading/unpacking pip from https://pypi.python.org/...
- ...
- Successfully installed pip setuptools
-
- % venv/bin/pip install tahoe-lafs
- Collecting tahoe-lafs
- ...
- Installing collected packages: ...
- Successfully installed ...
-
- % venv/bin/tahoe --version
- tahoe-lafs: 1.15.1
- foolscap: ...
-
- %
-
-Install From a Source Tarball
------------------------------
-
-You can also install directly from the source tarball URL. To verify
-signatures, first see verifying_signatures_ and replace the URL in the
-following instructions with the local filename.
-
- % virtualenv venv
- New python executable in ~/venv/bin/python2.7
- Installing setuptools, pip, wheel...done.
- 
- % venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2
- Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.1.tar.bz2
- ...
- Installing collected packages: ...
- Successfully installed ...
-
- % venv/bin/tahoe --version
- tahoe-lafs: 1.15.1
- ...
-
-.. _verifying_signatures:
-
-Verifying Signatures
---------------------
-
-First download the source tarball and then any signatures. There are several
-developers who are able to produce signatures for a release. A release may
-have multiple signatures. All should be valid and you should confirm at least
-one of them (ideally, confirm all).
-
-This statement, signed by the existing Tahoe release-signing key, attests to
-those developers authorized to sign a Tahoe release:
-
-.. include:: developer-release-signatures
-   :code:
-
-Signatures are made available beside the release. So for example, a release
-like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might
-have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and
-``tahoe-lafs-1.16.0.tar.bz2.warner.asc``.
-
-To verify the signatures using GnuPG::
-
-  % gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2
-  gpg: Signature made XXX
-  gpg:                using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7
-  gpg: Good signature from "meejah <meejah@meejah.ca>" [full]
-  % gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2
-  gpg: Signature made XXX
-  gpg:                using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF
-  gpg: Good signature from "Brian Warner <warner@lothar.com>" [full]
-
-
-
-Extras
-------
-
-Tahoe-LAFS provides some functionality only when explicitly requested at installation time.
-It does this using the "extras" feature of setuptools.
-You can request these extra features when running the ``pip install`` command like this::
-
-  % venv/bin/pip install tahoe-lafs[tor]
-
-This example enables support for listening and connecting using Tor.
-The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested.
-
-Hacking On Tahoe-LAFS
----------------------
-
-To modify the Tahoe source code, you should get a git checkout, and install
-with the ``--editable`` flag. You should also use the ``[test]`` extra to get
-the additional libraries needed to run the unit tests::
-
- % git clone https://github.com/tahoe-lafs/tahoe-lafs.git
-
- % cd tahoe-lafs
-
- % virtualenv venv
-
- % venv/bin/pip install --editable .[test]
- Obtaining file::~/tahoe-lafs
- ...
- Successfully installed ...
-
- % venv/bin/tahoe --version
- tahoe-lafs: 1.15.1
- ...
-
-This way, you won't have to re-run the ``pip install`` step each time you
-modify the source code.
-
-Running the ``tahoe`` executable
-================================
-
-The rest of the Tahoe-LAFS documentation assumes that you can run the
-``tahoe`` executable that you just created. You have four basic options:
-
-* Use the full path each time (e.g. ``~/venv/bin/tahoe``).
-* "`Activate`_" the virtualenv with ``. venv/bin/activate``, to get a
-  subshell with a ``$PATH`` that includes the ``venv/bin/`` directory, then
-  you can just run ``tahoe``.
-* Change your ``$PATH`` to include the ``venv/bin/`` directory, so you can
-  just run ``tahoe``.
-* Symlink from ``~/bin/tahoe`` to the ``tahoe`` executable. Since ``~/bin``
-  is typically in your ``$PATH`` (at least if it exists when you log in),
-  this will let you just run ``tahoe``.
-
-You might also find the `pipsi`_ tool convenient: ``pipsi install
-tahoe-lafs`` will create a new virtualenv, install tahoe into it, then
-symlink just the executable (into ``~/.local/bin/tahoe``). Then either add
-``~/.local/bin/`` to your ``$PATH``, or make one last symlink into
-``~/bin/tahoe``.
-
-.. _Activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script
-.. _pipsi: https://pypi.python.org/pypi/pipsi/0.9
-
-Running the Self-Tests
-======================
-
-To run the self-tests from a source tree, you'll need ``tox`` installed. On a
-Debian/Ubuntu system, use ``apt-get install tox``. You can also install it
-into your tahoe-specific virtualenv with ``pip install tox``.
-
-Then just run ``tox``. This will create a new fresh virtualenv, install Tahoe
-(from the source tree, including any changes you have made) and all its
-dependencies (including testing-only dependencies) into the virtualenv, then
-run the unit tests. This ensures that the tests are repeatable and match the
-results of other users, unaffected by any other Python packages installed on
-your machine. On a modern computer this will take 5-10 minutes, and should
-result in a "all tests passed" mesage::
-
- % tox
- GLOB sdist-make: ~/tahoe-lafs/setup.py
- py27 recreate: ~/tahoe-lafs/.tox/py27
- py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.15.1.zip
- py27 runtests: commands[0] | tahoe --version
- py27 runtests: commands[1] | trial --rterrors allmydata
- allmydata.test.test_auth
-   AccountFileCheckerKeyTests
-     test_authenticated ...                                           [OK]
-     test_missing_signature ...                                       [OK]
-  ...
- Ran 1186 tests in 423.179s
-
- PASSED (skips=7, expectedFailures=3, successes=1176)
- __________________________ summary ___________________________________
-   py27: commands succeeded
-   congratulations :)
-
-Common Problems
-===============
-
-If you see an error like ``fatal error: Python.h: No such file or directory``
-while compiling the dependencies, you need the Python development headers. If
-you are on a Debian or Ubuntu system, you can install them with ``sudo
-apt-get install python-dev``. On RedHat/Fedora, install ``python-devel``.
-
-Similar errors about ``openssl/crypto.h`` indicate that you are missing the
-OpenSSL development headers (``libssl-dev``). Likewise ``ffi.h`` means you
-need ``libffi-dev``.
-
-**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
-
-
-Using Tahoe-LAFS
-================
-
-Now you are ready to deploy a decentralized filesystem. You will use the
-``tahoe`` executable to create, configure, and launch your Tahoe-LAFS nodes.
-See :doc:`running` for instructions on how to do that.

docs/Installation/install-on-desert-island.rst

@@ -1,6 +1,6 @@
-******************************************
+***************************************
-How To Build Tahoe-LAFS On A Desert Island
+Building Tahoe-LAFS On A Desert Island
-******************************************
+***************************************
 
 (or an airplane, or anywhere else without internet connectivity)
 

docs/Installation/install-on-linux.rst

@@ -0,0 +1,75 @@
+****************************
+Building Tahoe-LAFS on Linux
+****************************
+
+Tahoe-LAFS has made packages available for installing on many linux and BSD distributions.
+Debian and Ubuntu users can use ``apt-get install tahoe-lafs``.
+If you are working on a Linux distribution which does not have Tahoe-LAFS or are looking to hack on the source code, you can build Tahoe-LAFS yourself:
+
+Prerequisites
+=============
+
+Make sure the following are installed:
+
+* **Python 3's latest version**: Check for the version by running ``python --version``.
+* **pip**: Most python installations already include ``pip``. However, if your installation does not, see `pip installation <https://pip.pypa.io/en/stable/installing/>`_.
+* **virtualenv**: Use ``pip`` to install virtualenv::
+  
+    pip install --user virtualenv
+
+* **C compiler and libraries**:
+
+    * ``python-dev``: Python development headers.
+    * ``libffi-dev``: Foreign Functions Interface library.
+    * ``libssl-dev``: SSL library, Tahoe-LAFS needs OpenSSL version 1.1.1c or greater.
+  
+    .. note::
+       If you are working on Debian or Ubuntu, you can install the necessary libraries using ``apt-get``::
+
+        apt-get install python-dev libffi-dev libssl-dev 
+
+       On an RPM-based system such as Fedora, you can install the necessary libraries using ``yum`` or ``rpm``. However, the packages may be named differently.
+
+Install the Latest Tahoe-LAFS Release
+=====================================
+
+If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS directly from source by creating a ``virtualenv`` instance:
+
+1. Clone the Tahoe-LAFS repository::
+   
+    git clone https://github.com/tahoe-lafs/tahoe-lafs.git
+
+2. Move into the tahoe-lafs directory::
+      
+    cd tahoe-lafs
+
+3. Create a fresh virtualenv for your Tahoe-LAFS install::
+   
+    virtualenv venv 
+
+.. note::
+   venv is the name of the virtual environment in this example. Use any name for your environment.
+
+4. Upgrade ``pip`` and ``setuptools`` on the newly created virtual environment::
+   
+    venv/bin/pip install -U pip setuptools
+
+5. If you'd like to modify the Tahoe source code, you need to install Tahoe-LAFS with the ``--editable`` flag with the ``test`` extra::
+   
+    venv/bin/pip install --editable .[test]
+
+.. note::
+   Tahoe-LAFS provides extra functionality when requested explicitly at installation using the "extras" feature of setuptools. To learn more about the extras which Tahoe supports, see Tahoe extras.
+
+6. Verify installation by checking for the version::
+   
+    venv/bin/tahoe --version
+
+If you do not want to use the full path, i.e., ``venv/bin/tahoe`` everytime you want to run tahoe, you can activate the ``virtualenv``::
+  
+   . venv/bin/activate
+
+  This will generate a subshell with a ``$PATH`` that includes the ``venv/bin/`` directory.
+
+
+

docs/Installation/install-on-windows.rst

@@ -0,0 +1,45 @@
+******************************
+Building Tahoe-LAFS on Windows
+******************************
+
+If you are looking to hack on the source code or run pre-release code, we recommend you create a virtualenv instance and install Tahoe-LAFS into that:
+
+
+1. Make sure you have Powershell installed. See `PowerShell installation <https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-7.1>`_.
+
+2. Install the latest version of Python 3. Download the .exe file at the `python website <https://www.python.org/downloads/>`_.
+
+3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**.
+
+4. Start PowerShell and enter the following command to verify python installation::
+   
+    python --version
+
+5. Use ``pip`` to install ``virtualenv``::
+   
+    pip install --user virtualenv
+
+6. Create a fresh virtualenv for your Tahoe-LAFS install using the following command::
+    
+     virtualenv venv
+
+ .. note::
+    venv is the name of the virtual environment in this example. Use any name for your environment.
+
+7. Use pip to install Tahoe-LAFS in the virtualenv instance::
+   
+    venv\Scripts\pip install tahoe-lafs
+
+6. Verify installation by checking for the version::
+   
+    venv\Scripts\tahoe --version
+
+If you do not want to use the full path, i.e. ``venv\Scripts\tahoe`` everytime you want to run tahoe, you can:
+
+* Activate the virtualenv::
+  
+   . venv\Scripts\activate
+   
+  This will generate a subshell with a ``$PATH`` that includes the ``venv\Scripts\`` directory.
+
+* Change your ``$PATH`` to include the ``venv\Scripts`` directory.

docs/Installation/install-tahoe.rst

@@ -0,0 +1,68 @@
+.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*-
+
+..
+    note: if you aren't reading the rendered form of these docs at
+    http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any
+    ":doc:" links refer to other files in this docs/ directory
+
+*********************
+Installing Tahoe-LAFS
+*********************
+
+`Tahoe-LAFS`_ is a secure, decentralized, and fault-tolerant storage system.
+To see an overview of the architecture and security properties, see :doc:`Welcome to Tahoe LAFS! <../about-tahoe>`
+
+Tahoe-LAFS can be installed and used on any of the following operating systems.
+
+.. _Tahoe-LAFS: https://tahoe-lafs.org
+
+Microsoft Windows
+=================
+
+To install Tahoe-LAFS on Windows:
+
+1. Make sure you have Powershell installed. See `PowerShell installation <https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-7.1>`_.
+
+2. Install the latest version of Python 3. Download the .exe file at the `python website <https://www.python.org/downloads/>`_.
+
+3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**.
+
+4. Start PowerShell and enter the following command to verify python installation::
+   
+    python --version
+
+5. Enter the following command to install Tahoe-LAFS::
+   
+    pip install tahoe-lafs
+
+6. Verify installation by checking for the version::
+   
+    tahoe --version
+
+If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`.
+
+Linux, BSD, or MacOS
+====================
+
+Tahoe-LAFS can be installed on MacOS, many Linux and BSD distributions. If you are using Ubuntu or Debian, run the following command to install Tahoe-LAFS::
+
+ apt-get install tahoe-lafs
+
+If you are working on MacOS or a Linux distribution which does not have Tahoe-LAFS packages, you can build it yourself:
+
+1. Make sure the following are installed:
+
+   * **Python 3's latest version**: Check for the version by running ``python --version``.
+   * **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation <https://pip.pypa.io/en/stable/installing/>`_.
+
+2. Install Tahoe-LAFS using pip::
+   
+    pip install tahoe-lafs
+
+3. Verify installation by checking for the version::
+   
+    tahoe --version
+
+If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`.    
+
+You can always write to the `tahoe-dev mailing list <https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev>`_ or chat on the `Libera.chat IRC <irc://irc.libera.chat/%23tahoe-lafs>`_ if you are not able to get Tahoe-LAFS up and running on your deployment.

docs/OS-X.rst

@@ -1,23 +0,0 @@
-==============
-OS-X Packaging
-==============
-
-Pre-built Tahoe-LAFS ".pkg" installers for OS-X are generated with each
-source-code commit. These installers offer an easy way to get Tahoe and all
-its dependencies installed on your Mac. They do not yet provide a
-double-clickable application: after installation, you will have a "tahoe"
-command-line tool, which you can use from a shell (a Terminal window) just as
-if you'd installed from source.
-
-Installers are available from this directory:
-
- https://tahoe-lafs.org/source/tahoe-lafs/tarballs/OS-X-packages/
-
-Download the latest .pkg file to your computer and double-click on it. This
-will install to /Applications/tahoe.app, however the app icon there is not
-how you use Tahoe (launching it will get you a dialog box with a reminder to
-use Terminal). ``/Applications/tahoe.app/bin/tahoe`` is the executable. The
-next shell you start ought to have that directory in your $PATH (thanks to a
-file in ``/etc/paths.d/``), unless your ``.profile`` overrides it.
-
-Tahoe-LAFS is also easy to install with pip, as described in the README.

docs/about-tahoe.rst

@@ -127,7 +127,7 @@ For more technical detail, please see the `the doc page`_ on the Wiki.
 Get Started
 ===========
 
-To use Tahoe-LAFS, please see :doc:`INSTALL`.
+To use Tahoe-LAFS, please see :doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`.
 
 License
 =======

docs/index.rst

@@ -22,6 +22,13 @@ preserving your privacy and security.
    :caption: Using Tahoe-LAFS
 
    INSTALL
+
+   about-tahoe
+   Installation/install-tahoe
+   Installation/install-on-windows
+   Installation/install-on-linux
+   Installation/install-on-desert-island
+
    running
    configuration
    servers
@@ -50,6 +57,8 @@ preserving your privacy and security.
    performance
    logging
    stats
+   debian
+   build/build-pyOpenSSL
 
 .. toctree::
    :maxdepth: 1

docs/release-checklist.rst

@@ -70,7 +70,7 @@ Create Branch and Apply Updates
   - commit it
 
 - update "docs/known_issues.rst" if appropriate
-- update "docs/INSTALL.rst" references to the new release
+- update "docs/Installation/install-tahoe.rst" references to the new release
 - Push the branch to github
 - Create a (draft) PR; this should trigger CI (note that github
   doesn't let you create a PR without some changes on the branch so

docs/running.rst

@@ -10,7 +10,7 @@ Introduction
 
 This is how to run a Tahoe-LAFS client or a complete Tahoe-LAFS grid.
 First you have to install the Tahoe-LAFS software, as documented in
-:doc:`INSTALL`.
+:doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`.
 
 The ``tahoe`` program in your virtualenv's ``bin`` directory is used to
 create, start, and stop nodes. Each node lives in a separate base

docs/windows.rst

@@ -1,83 +0,0 @@
-Building Tahoe-LAFS on Windows
-==============================
-
-You'll need ``python``, ``pip``, and ``virtualenv``. But you won't need a
-compiler.
-
-Preliminaries
--------------
-
-1: Install Python-2.7.11 . Use the "Windows x86-64 MSI installer" at
-https://www.python.org/downloads/release/python-2711/
-
-2: That should install ``pip``, but if it doesn't, look at
-https://pip.pypa.io/en/stable/installing/ for installation instructions.
-
-3: Install ``virtualenv`` with
-https://virtualenv.pypa.io/en/latest/installation.html
-
-Installation
-------------
-
-1: Start a CLI shell (e.g. PowerShell)
-
-2: Create a new virtualenv. Everything specific to Tahoe will go into this.
-You can use whatever name you like for the virtualenv, but example uses
-"venv"::
-
-    PS C:\Users\me> virtualenv venv
-    New python executable in C:\Users\me\venv\Scripts\python.exe
-    Installing setuptools, pip, wheel...done.
-    >
-
-3: Use the virtualenv's ``pip`` to install the latest release of Tahoe-LAFS
-into this virtualenv::
-
-    PS C:\Users\me> venv\Scripts\pip install tahoe-lafs
-    Collecting tahoe-lafs
-    ...
-    Installing collected packages: ...
-    Successfully installed ...
-    >
-
-4: Verify that Tahoe was installed correctly by running ``tahoe --version``,
-using the ``tahoe`` from the virtualenv's Scripts directory::
-
-    PS C:\Users\me> venv\Scripts\tahoe --version
-    tahoe-lafs: 1.11
-    foolscap: ...
-
-Running Tahoe-LAFS
-------------------
-
-The rest of the documentation assumes you can run the ``tahoe`` executable
-just as you did in step 4 above. If you want to type just ``tahoe`` instead
-of ``venv\Scripts\tahoe``, you can either "`activate`_" the virtualenv (by
-running ``venv\Scripts\activate``, or you can add the Scripts directory to
-your ``%PATH%`` environment variable.
-
-Now use the docs in :doc:`running` to learn how to configure your first
-Tahoe node.
-
-.. _activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script
-
-Installing A Different Version
-------------------------------
-
-The ``pip install tahoe-lafs`` command above will install the latest release
-(from PyPI). If instead, you want to install from a git checkout, then run
-the following command (using pip from the virtualenv, from the root of your
-git checkout)::
-
-    $ venv\Scripts\pip install .
-
-If you're planning to hack on the source code, you might want to add
-``--editable`` so you won't have to re-install each time you make a change.
-
-Dependencies
-------------
-
-Tahoe-LAFS depends upon several packages that use compiled C code (such as zfec).
-This code must be built separately for each platform (Windows, OS-X, and different flavors of Linux).
-Fortunately, this is now done by upstream packages for most platforms.
-The result is that a C compiler is usually not required to install Tahoe-LAFS.

misc/coding_tools/check-debugging.py

@@ -1,13 +1,18 @@
 #! /usr/bin/python
 
-# ./check-debugging.py src
+"""
+Checks for defer.setDebugging().
+
+Runs on Python 3.
+
+Usage: ./check-debugging.py src
+"""
 
 from __future__ import print_function
 
 import sys, re, os
 
 ok = True
-umids = {}
 
 for starting_point in sys.argv[1:]:
     for root, dirs, files in os.walk(starting_point):

misc/coding_tools/check-miscaptures.py

@@ -1,186 +0,0 @@
-#! /usr/bin/python
-
-from __future__ import print_function
-
-import os, sys, compiler
-from compiler.ast import Node, For, While, ListComp, AssName, Name, Lambda, Function
-
-
-def check_source(source):
-    return check_thing(compiler.parse, source)
-
-def check_file(path):
-    return check_thing(compiler.parseFile, path)
-
-def check_thing(parser, thing):
-    try:
-        ast = parser(thing)
-    except SyntaxError as e:
-        return e
-    else:
-        results = []
-        check_ast(ast, results)
-        return results
-
-def check_ast(ast, results):
-    """Check a node outside a loop."""
-    if isinstance(ast, (For, While, ListComp)):
-        check_loop(ast, results)
-    else:
-        for child in ast.getChildNodes():
-            if isinstance(ast, Node):
-                check_ast(child, results)
-
-def check_loop(ast, results):
-    """Check a particular outer loop."""
-
-    # List comprehensions have a poorly designed AST of the form
-    # ListComp(exprNode, [ListCompFor(...), ...]), in which the
-    # result expression is outside the ListCompFor node even though
-    # it is logically inside the loop(s).
-    # There may be multiple ListCompFor nodes (in cases such as
-    #   [lambda: (a,b) for a in ... for b in ...]
-    # ), and that case they are not nested in the AST. But these
-    # warts (nonobviously) happen not to matter for our analysis.
-
-    assigned = {}  # maps name to lineno of topmost assignment
-    nested = set()
-    collect_assigned_and_nested(ast, assigned, nested)
-
-    # For each nested function...
-    for funcnode in nested:
-        # Check for captured variables in this function.
-        captured = set()
-        collect_captured(funcnode, assigned, captured, False)
-        for name in captured:
-            # We want to report the outermost capturing function
-            # (since that is where the workaround will need to be
-            # added), and the topmost assignment to the variable.
-            # Just one report per capturing function per variable
-            # will do.
-            results.append(make_result(funcnode, name, assigned[name]))
-
-        # Check each node in the function body in case it
-        # contains another 'for' loop.
-        childnodes = funcnode.getChildNodes()[len(funcnode.defaults):]
-        for child in childnodes:
-            check_ast(child, results)
-
-def collect_assigned_and_nested(ast, assigned, nested):
-    """
-    Collect the names assigned in this loop, not including names
-    assigned in nested functions. Also collect the nodes of functions
-    that are nested one level deep.
-    """
-    if isinstance(ast, AssName):
-        if ast.name not in assigned or assigned[ast.name] > ast.lineno:
-            assigned[ast.name] = ast.lineno
-    else:
-        childnodes = ast.getChildNodes()
-        if isinstance(ast, (Lambda, Function)):
-            nested.add(ast)
-
-            # The default argument expressions are "outside" the
-            # function, even though they are children of the
-            # Lambda or Function node.
-            childnodes = childnodes[:len(ast.defaults)]
-
-        for child in childnodes:
-            if isinstance(ast, Node):
-                collect_assigned_and_nested(child, assigned, nested)
-
-def collect_captured(ast, assigned, captured, in_function_yet):
-    """Collect any captured variables that are also in assigned."""
-    if isinstance(ast, Name):
-        if ast.name in assigned:
-            captured.add(ast.name)
-    else:
-        childnodes = ast.getChildNodes()
-        if isinstance(ast, (Lambda, Function)):
-            # Formal parameters of the function are excluded from
-            # captures we care about in subnodes of the function body.
-            new_assigned = assigned.copy()
-            remove_argnames(ast.argnames, new_assigned)
-
-            if len(new_assigned) > 0:
-                for child in childnodes[len(ast.defaults):]:
-                    collect_captured(child, new_assigned, captured, True)
-
-            # The default argument expressions are "outside" *this*
-            # function, even though they are children of the Lambda or
-            # Function node.
-            if not in_function_yet:
-                return
-            childnodes = childnodes[:len(ast.defaults)]
-
-        for child in childnodes:
-            if isinstance(ast, Node):
-                collect_captured(child, assigned, captured, True)
-
-
-def remove_argnames(names, fromset):
-    for element in names:
-        if element in fromset:
-            del fromset[element]
-        elif isinstance(element, (tuple, list)):
-            remove_argnames(element, fromset)
-
-
-def make_result(funcnode, var_name, var_lineno):
-    if hasattr(funcnode, 'name'):
-        func_name = 'function %r' % (funcnode.name,)
-    else:
-        func_name = '<lambda>'
-    return (funcnode.lineno, func_name, var_name, var_lineno)
-
-def report(out, path, results):
-    for r in results:
-        print(path + (":%r %s captures %r assigned at line %d" % r), file=out)
-
-def check(sources, out):
-    class Counts(object):
-        n = 0
-        processed_files = 0
-        suspect_files = 0
-        error_files = 0
-    counts = Counts()
-
-    def _process(path):
-        results = check_file(path)
-        if isinstance(results, SyntaxError):
-            print(path + (" NOT ANALYSED due to syntax error: %s" % results), file=out)
-            counts.error_files += 1
-        else:
-            report(out, path, results)
-            counts.n += len(results)
-            counts.processed_files += 1
-            if len(results) > 0:
-                counts.suspect_files += 1
-
-    for source in sources:
-        print("Checking %s..." % (source,), file=out)
-        if os.path.isfile(source):
-            _process(source)
-        else:
-            for (dirpath, dirnames, filenames) in os.walk(source):
-                for fn in filenames:
-                    (basename, ext) = os.path.splitext(fn)
-                    if ext == '.py':
-                        _process(os.path.join(dirpath, fn))
-
-    print("%d suspiciously captured variables in %d out of %d file(s)."
-                  % (counts.n, counts.suspect_files, counts.processed_files), file=out)
-    if counts.error_files > 0:
-        print("%d file(s) not processed due to syntax errors."
-                      % (counts.error_files,), file=out)
-    return counts.n
-
-
-sources = ['src']
-if len(sys.argv) > 1:
-    sources = sys.argv[1:]
-if check(sources, sys.stderr) > 0:
-    sys.exit(1)
-
-
-# TODO: self-tests

misc/coding_tools/check-umids.py

@@ -1,4 +1,10 @@
-#! /usr/bin/python
+#! /usr/bin/python3
+
+"""
+Ensure UMIDS are unique.
+
+This runs on Python 3.
+"""
 
 # ./check-umids.py src
 

misc/operations_helpers/cpu-watcher.tac

@@ -201,7 +201,9 @@ class CPUWatcher(service.MultiService, resource.Resource, Referenceable):
                 log.msg("error reading process %s (%s), ignoring" % (pid, name))
                 log.err()
         try:
-            pickle.dump(self.history, open("history.pickle.tmp", "wb"))
+            # Newer protocols won't work in Python 2; when it is dropped,
+            # protocol v4 can be used (added in Python 3.4).
+            pickle.dump(self.history, open("history.pickle.tmp", "wb"), protocol=2)
             os.rename("history.pickle.tmp", "history.pickle")
         except:
             pass

newsfragments/3681.installation

@@ -1,3 +0,0 @@
-Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has
-removed vcpython27 compiler downloads from their site, and Tahoe-LAFS
-needs vcpython27 to build and install netifaces on 64-bit Windows.

newsfragments/3681.minor

@@ -0,0 +1,8 @@
+(The below text is no longer valid: netifaces has released a 64-bit
+Python 2.7 wheel for Windows.  Ticket #3733 made the switch in CI.  We
+should be able to test and run Tahoe-LAFS without needing vcpython27
+now.)
+
+Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has
+removed vcpython27 compiler downloads from their site, and Tahoe-LAFS
+needs vcpython27 to build and install netifaces on 64-bit Windows.

newsfragments/3733.installation

@@ -0,0 +1 @@
+Use netifaces 0.11.0 wheel package from PyPI.org if you use 64-bit Python 2.7 on Windows.  VCPython27 downloads are no longer available at Microsoft's website, which has made building Python 2.7 wheel packages of Python libraries with C extensions (such as netifaces) on Windows difficult.

newsfragments/3738.bugfix

@@ -0,0 +1 @@
+Fix regression where uploading files with non-ASCII names failed.

newsfragments/3739.bugfix

@@ -0,0 +1 @@
+Fixed annoying UnicodeWarning message on Python 2 when running CLI tools.

newsfragments/3747.documentation

@@ -0,0 +1 @@
+Rewriting the installation guide for Tahoe-LAFS.

setup.py

@@ -54,9 +54,9 @@ install_requires = [
     # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo
     # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs
     # * foolscap 0.13.2 drops i2p support completely
-    # * foolscap >= 20.4 is necessary for Python 3
+    # * foolscap >= 21.7 is necessary for Python 3 with i2p support.
     "foolscap == 0.13.1 ; python_version < '3.0'",
-    "foolscap >= 20.4.0 ; python_version > '3.0'",
+    "foolscap >= 21.7.0 ; python_version > '3.0'",
 
     # * cryptography 2.6 introduced some ed25519 APIs we rely on.  Note that
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
@@ -114,12 +114,11 @@ install_requires = [
 
     # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped
     # Python 2 entirely; stick to the version known to work for us.
-    # XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404
+    "pyrsistent < 0.17.0 ; python_version < '3.0'",
-    "pyrsistent < 0.17.0",
+    "pyrsistent ; python_version > '3.0'",
 
     # A great way to define types of values.
-    # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390
+    "attrs >= 18.2.0",
-    "attrs >= 18.2.0, < 20",
 
     # WebSocket library for twisted and asyncio
     "autobahn >= 19.5.2",

src/allmydata/frontends/auth.py

@@ -1,3 +1,15 @@
+"""
+Authentication for frontends.
+"""
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from zope.interface import implementer
 from twisted.internet import defer
 from twisted.cred import error, checkers, credentials

src/allmydata/scripts/common_http.py

@@ -94,11 +94,17 @@ def do_http(method, url, body=b""):
 
 
 def format_http_success(resp):
-    return "%s %s" % (resp.status, quote_output(resp.reason, quotemarks=False))
+    # ensure_text() shouldn't be necessary when Python 2 is dropped.
+    return quote_output(
+        "%s %s" % (resp.status, six.ensure_text(resp.reason)),
+        quotemarks=False)
 
 def format_http_error(msg, resp):
-    return "%s: %s %s\n%s" % (msg, resp.status, quote_output(resp.reason, quotemarks=False),
+    # ensure_text() shouldn't be necessary when Python 2 is dropped.
-                              quote_output(resp.read(), quotemarks=False))
+    return quote_output(
+        "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason),
+                           six.ensure_text(resp.read())),
+        quotemarks=False)
 
 def check_http_error(resp, stderr):
     if resp.status < 200 or resp.status >= 300:

src/allmydata/scripts/runner.py

@@ -204,7 +204,8 @@ def _setup_coverage(reactor):
     """
     # can we put this _setup_coverage call after we hit
     # argument-parsing?
-    if '--coverage' not in sys.argv:
+    # ensure_str() only necessary on Python 2.
+    if six.ensure_str('--coverage') not in sys.argv:
         return
     sys.argv.remove('--coverage')
 

src/allmydata/storage/crawler.py

@@ -252,7 +252,9 @@ class ShareCrawler(service.MultiService):
         self.state["last-complete-prefix"] = last_complete_prefix
         tmpfile = self.statefile + ".tmp"
         with open(tmpfile, "wb") as f:
-            pickle.dump(self.state, f)
+            # Newer protocols won't work in Python 2; when it is dropped,
+            # protocol v4 can be used (added in Python 3.4).
+            pickle.dump(self.state, f, protocol=2)
         fileutil.move_into_place(tmpfile, self.statefile)
 
     def startService(self):

src/allmydata/storage/expirer.py

@@ -95,7 +95,9 @@ class LeaseCheckingCrawler(ShareCrawler):
         if not os.path.exists(self.historyfile):
             history = {} # cyclenum -> dict
             with open(self.historyfile, "wb") as f:
-                pickle.dump(history, f)
+                # Newer protocols won't work in Python 2; when it is dropped,
+                # protocol v4 can be used (added in Python 3.4).
+                pickle.dump(history, f, protocol=2)
 
     def create_empty_cycle_dict(self):
         recovered = self.create_empty_recovered_dict()
@@ -319,7 +321,9 @@ class LeaseCheckingCrawler(ShareCrawler):
             oldcycles = sorted(history.keys())
             del history[oldcycles[0]]
         with open(self.historyfile, "wb") as f:
-            pickle.dump(history, f)
+            # Newer protocols won't work in Python 2; when it is dropped,
+            # protocol v4 can be used (added in Python 3.4).
+            pickle.dump(history, f, protocol=2)
 
     def get_state(self):
         """In addition to the crawler state described in

src/allmydata/test/cli/test_cp.py

@@ -55,6 +55,11 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
         d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1))
 
+        # Version where destination filename is explicitly Unicode too.
+        d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:" + artonwall_arg + "-2"))
+        d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg + "-2"))
+        d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1))
+
         d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
 
         d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
@@ -74,7 +79,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 self.failUnlessReallyEqual(rc, 0)
                 if PY2:
                     out = out.decode(get_io_encoding())
-                self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n")
+                self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n\u00C4rtonwall-2\n")
                 self.assertEqual(len(err), 0, err)
         d.addCallback(_check)
 

src/allmydata/test/test_auth.py

@@ -39,8 +39,10 @@ dBSD8940XU3YW+oeq8e+p3yQ2GinHfeJ3BYQyNQLuMAJ
 """)
 
 DUMMY_ACCOUNTS = u"""\
-alice password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111
+alice herpassword URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111
 bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222
+
+# dennis password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111
 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333
 """.format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii")
 
@@ -54,7 +56,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
         abspath = abspath_expanduser_unicode(str(self.account_file.path))
         self.checker = auth.AccountFileChecker(None, abspath)
 
-    def test_unknown_user(self):
+    def test_unknown_user_ssh(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with
         UnauthorizedLogin if called with an SSHPrivateKey object with a
@@ -65,6 +67,19 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
         avatarId = self.checker.requestAvatarId(key_credentials)
         return self.assertFailure(avatarId, error.UnauthorizedLogin)
 
+    def test_unknown_user_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        UnauthorizedLogin if called with an SSHPrivateKey object with a
+        username not present in the account file.
+
+        We use a commented out user, so we're also checking that comments are
+        skipped.
+        """
+        key_credentials = credentials.UsernamePassword(b"dennis", b"password")
+        d = self.checker.requestAvatarId(key_credentials)
+        return self.assertFailure(d, error.UnauthorizedLogin)
+
     def test_password_auth_user_with_ssh_key(self):
         """
         AccountFileChecker.requestAvatarId returns a Deferred that fires with
@@ -81,7 +96,21 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
         AccountFileChecker.requestAvatarId returns a Deferred that fires with
         the user if the correct password is given.
         """
-        key_credentials = credentials.UsernamePassword(b"alice", b"password")
+        key_credentials = credentials.UsernamePassword(b"alice", b"herpassword")
+        d = self.checker.requestAvatarId(key_credentials)
+        def authenticated(avatarId):
+            self.assertEqual(
+                (b"alice",
+                 b"URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111"),
+                (avatarId.username, avatarId.rootcap))
+        return d
+
+    def test_password_auth_user_with_correct_hashed_password(self):
+        """
+        AccountFileChecker.requestAvatarId returns a Deferred that fires with
+        the user if the correct password is given in hashed form.
+        """
+        key_credentials = credentials.UsernameHashedPassword(b"alice", b"herpassword")
         d = self.checker.requestAvatarId(key_credentials)
         def authenticated(avatarId):
             self.assertEqual(

src/allmydata/util/_python3.py

@@ -52,6 +52,7 @@ PORTED_MODULES = [
     "allmydata.deep_stats",
     "allmydata.dirnode",
     "allmydata.frontends",
+    "allmydata.frontends.auth",
     "allmydata.frontends.sftpd",
     "allmydata.hashtree",
     "allmydata.history",

src/allmydata/util/jsonbytes.py

@@ -28,8 +28,8 @@ if PY2:
     codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2)
 
 
-def _bytes_to_unicode(any_bytes, obj):
+def bytes_to_unicode(any_bytes, obj):
-    """Create a function that recursively converts bytes to unicode.
+    """Convert bytes to unicode.
 
     :param any_bytes: If True, also support non-UTF-8-encoded bytes.
     :param obj: Object to de-byte-ify.
@@ -63,11 +63,11 @@ class UTF8BytesJSONEncoder(json.JSONEncoder):
     """
     def encode(self, o, **kwargs):
         return json.JSONEncoder.encode(
-            self, _bytes_to_unicode(False, o), **kwargs)
+            self, bytes_to_unicode(False, o), **kwargs)
 
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
-            self, _bytes_to_unicode(False, o), **kwargs)
+            self, bytes_to_unicode(False, o), **kwargs)
 
 
 class AnyBytesJSONEncoder(json.JSONEncoder):
@@ -79,11 +79,11 @@ class AnyBytesJSONEncoder(json.JSONEncoder):
     """
     def encode(self, o, **kwargs):
         return json.JSONEncoder.encode(
-            self, _bytes_to_unicode(True, o), **kwargs)
+            self, bytes_to_unicode(True, o), **kwargs)
 
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
-            self, _bytes_to_unicode(True, o), **kwargs)
+            self, bytes_to_unicode(True, o), **kwargs)
 
 
 def dumps(obj, *args, **kwargs):

src/allmydata/util/log.py

@@ -18,6 +18,16 @@ from pyutil import nummedobj
 from foolscap.logging import log
 from twisted.python import log as tw_log
 
+if PY2:
+    def bytes_to_unicode(ign, obj):
+        return obj
+else:
+    # We want to convert bytes keys to Unicode, otherwise JSON serialization
+    # inside foolscap will fail (for details see
+    # https://github.com/warner/foolscap/issues/88)
+    from .jsonbytes import bytes_to_unicode
+
+
 NOISY = log.NOISY # 10
 OPERATIONAL = log.OPERATIONAL # 20
 UNUSUAL = log.UNUSUAL # 23
@@ -28,7 +38,8 @@ SCARY = log.SCARY # 35
 BAD = log.BAD # 40
 
 
-msg = log.msg
+def msg(*args, **kwargs):
+    return log.msg(*args, **bytes_to_unicode(True, kwargs))
 
 # If log.err() happens during a unit test, the unit test should fail. We
 # accomplish this by sending it to twisted.log too. When a WEIRD/SCARY/BAD
@@ -39,7 +50,7 @@ def err(failure=None, _why=None, **kwargs):
     tw_log.err(failure, _why, **kwargs)
     if 'level' not in kwargs:
         kwargs['level'] = log.UNUSUAL
-    return log.err(failure, _why, **kwargs)
+    return log.err(failure, _why, **bytes_to_unicode(True, kwargs))
 
 class LogMixin(object):
     """ I remember a msg id and a facility and pass them to log.msg() """
@@ -57,7 +68,8 @@ class LogMixin(object):
             if pmsgid is None:
                 pmsgid = self._grandparentmsgid
         kwargs = {ensure_str(k): v for (k, v) in kwargs.items()}
-        msgid = log.msg(msg, facility=facility, parent=pmsgid, *args, **kwargs)
+        msgid = log.msg(msg, facility=facility, parent=pmsgid, *args,
+                        **bytes_to_unicode(True, kwargs))
         if self._parentmsgid is None:
             self._parentmsgid = msgid
         return msgid

src/allmydata/web/common.py

@@ -90,6 +90,7 @@ from allmydata.util.time_format import (
 )
 from allmydata.util.encodingutil import (
     quote_output,
+    quote_output_u,
     to_bytes,
 )
 from allmydata.util import abbreviate
@@ -324,7 +325,7 @@ def humanize_exception(exc):
         return ("There was already a child by that name, and you asked me "
                 "to not replace it.", http.CONFLICT)
     if isinstance(exc, NoSuchChildError):
-        quoted_name = quote_output(exc.args[0], encoding="utf-8", quotemarks=False)
+        quoted_name = quote_output_u(exc.args[0], quotemarks=False)
         return ("No such child: %s" % quoted_name, http.NOT_FOUND)
     if isinstance(exc, NotEnoughSharesError):
         t = ("NotEnoughSharesError: This indicates that some "

tox.ini

@@ -104,17 +104,15 @@ setenv =
 commands =
          python --version
          # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-         python3 -b -m pytest --timeout=1800 --coverage -v {posargs:integration}
+         py.test --timeout=1800 --coverage -v {posargs:integration}
          coverage combine
          coverage report
 
 
+# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2
+# since that can give different results than flake8 on Python 3.
 [testenv:codechecks]
 basepython = python2.7
-# On macOS, git inside of towncrier needs $HOME.
-passenv = HOME
-whitelist_externals =
-         /bin/mv
 setenv =
          # Workaround an error when towncrier is run under the VCS hook,
          # https://stackoverflow.com/a/4027726/624787:
@@ -128,26 +126,35 @@ setenv =
 	 DEFAULT_FILES=src integration static misc setup.py
 commands =
          flake8 {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/check-miscaptures.py {posargs:{env:DEFAULT_FILES}}
-
-         # If towncrier.check fails, you forgot to add a towncrier news
-         # fragment explaining the change in this branch.  Create one at
-         # `newsfragments/<ticket>.<change type>` with some text for the news
-         # file.  See towncrier.pyproject.toml for legal <change type> values.
-         python -m towncrier.check --config towncrier.pyproject.toml
 
 
 [testenv:codechecks3]
 basepython = python3
+deps =
+     # Newer versions of PyLint have buggy configuration
+     # (https://github.com/PyCQA/pylint/issues/4574), so stick to old version
+     # for now.
+     pylint < 2.5
+# On macOS, git inside of towncrier needs $HOME.
+passenv = HOME
 setenv =
 	 # If no positional arguments are given, try to run the checks on the
 	 # entire codebase, including various pieces of supporting code.
 	 DEFAULT_FILES=src integration static misc setup.py
 commands =
          flake8 {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
+         # PyLint has other useful checks, might want to enable them:
+         # http://pylint.pycqa.org/en/latest/technical_reference/features.html
+         pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}}
+
+         # If towncrier.check fails, you forgot to add a towncrier news
+         # fragment explaining the change in this branch.  Create one at
+         # `newsfragments/<ticket>.<change type>` with some text for the news
+         # file.  See towncrier.toml for legal <change type> values.
+         python -m towncrier.check --config towncrier.toml
 
 
 [testenv:typechecks]
@@ -155,7 +162,11 @@ basepython = python3
 skip_install = True
 deps =
     mypy
-    git+https://github.com/Shoobx/mypy-zope
+    mypy-zope
+    types-mock
+    types-six
+    types-PyYAML
+    types-pkg_resources
     git+https://github.com/warner/foolscap
     # Twisted 21.2.0 introduces some type hints which we are not yet
     # compatible with.
@@ -166,62 +177,26 @@ commands = mypy src
 
 [testenv:draftnews]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
-# see comment in [testenv] about "certifi"
-whitelist_externals = mv
 deps =
-     certifi
+    # see comment in [testenv] about "certifi"
-     towncrier >= 19.2
+    certifi
+    towncrier==21.3.0
 commands =
-         # With pip >= 10 the existence of pyproject.toml (which we are
+    python -m towncrier --draft --config towncrier.toml
-         # required to have to configure towncrier) triggers a "build
-         # isolation" mode which prevents anything from working.  Avoid
-         # triggering that pip behavior by keeping the towncrier configuration
-         # somewhere else and only bringing it in when it's actually needed
-         # (after pip is done).
-         #
-         # Some discussion is available at
-         # https://github.com/pypa/pip/issues/5696
-         #
-         # towncrier post 19.2 (unreleased as of this writing) adds a --config
-         # option that can be used instead of this file shuffling.
-         mv towncrier.pyproject.toml pyproject.toml
-
-         # towncrier 19.2 + works with python2.7
-         python -m towncrier --draft
-
-         # put it back
-         mv pyproject.toml towncrier.pyproject.toml
 
 [testenv:news]
-passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
+# On macOS, git invoked from Tox needs $HOME.
-# see comment in [testenv] about "certifi"
+passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HOME
-whitelist_externals = mv
+whitelist_externals =
+    git
 deps =
-     certifi
+    # see comment in [testenv] about "certifi"    
-     towncrier >= 19.2
+    certifi
+    towncrier==21.3.0
 commands =
-         # With pip >= 10 the existence of pyproject.toml (which we are
+    python -m towncrier --yes --config towncrier.toml
-         # required to have to configure towncrier) triggers a "build
+    # commit the changes
-         # isolation" mode which prevents anything from working.  Avoid
+    git commit -m "update NEWS.txt for release"
-         # triggering that pip behavior by keeping the towncrier configuration
-         # somewhere else and only bringing it in when it's actually needed
-         # (after pip is done).
-         #
-         # Some discussion is available at
-         # https://github.com/pypa/pip/issues/5696
-         #
-         # towncrier post 19.2 (unreleased as of this writing) adds a --config
-         # option that can be used instead of this file shuffling.
-         mv towncrier.pyproject.toml pyproject.toml
-
-         # towncrier 19.2 + works with python2.7
-         python -m towncrier --yes
-
-         # put it back
-         mv pyproject.toml towncrier.pyproject.toml
-
-         # commit the changes
-         git commit -m "update NEWS.txt for release"
 
 [testenv:deprecations]
 commands =
@@ -281,8 +256,7 @@ deps =
     # PyInstaller 4.0 drops Python 2 support.  When we finish porting to
     # Python 3 we can reconsider this constraint.
     pyinstaller < 4.0
-    # 2021.5.13 broke on Windows. See https://github.com/erocarrera/pefile/issues/318
+    pefile ; platform_system == "Windows"
-    pefile < 2021.5.13 ; platform_system == "Windows"
 # Setting PYTHONHASHSEED to a known value assists with reproducible builds.
 # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build
 setenv=PYTHONHASHSEED=1