Merge remote-tracking branch 'origin/master' into 3528.test_cli-no-mock

This commit is contained in:
Jean-Paul Calderone 2021-08-11 12:56:00 -04:00
commit 66d40527d1
372 changed files with 4672 additions and 3671 deletions

View File

@ -15,79 +15,64 @@ workflows:
ci:
jobs:
# Start with jobs testing various platforms.
# Every job that pulls a Docker image from Docker Hub needs to provide
# credentials for that pull operation to avoid being subjected to
# unauthenticated pull limits shared across all of CircleCI. Use this
# first job to define a yaml anchor that can be used to supply a
# CircleCI job context which makes Docker Hub credentials available in
# the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "debian-9": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "debian-9":
{}
- "debian-10":
<<: *DOCKERHUB_CONTEXT
requires:
- "debian-9"
- "ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
{}
- "ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "ubuntu-16-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "fedora-29":
<<: *DOCKERHUB_CONTEXT
{}
- "fedora-28":
<<: *DOCKERHUB_CONTEXT
requires:
- "fedora-29"
- "centos-8":
<<: *DOCKERHUB_CONTEXT
{}
- "nixos-19-09":
<<: *DOCKERHUB_CONTEXT
{}
# Test against PyPy 2.7
- "pypy27-buster":
<<: *DOCKERHUB_CONTEXT
{}
# Just one Python 3.6 configuration while the port is in-progress.
- "python36":
<<: *DOCKERHUB_CONTEXT
{}
# Other assorted tasks and configurations
- "lint":
<<: *DOCKERHUB_CONTEXT
{}
- "pyinstaller":
<<: *DOCKERHUB_CONTEXT
{}
- "deprecations":
<<: *DOCKERHUB_CONTEXT
{}
- "c-locale":
<<: *DOCKERHUB_CONTEXT
{}
# Any locale other than C or UTF-8.
- "another-locale":
<<: *DOCKERHUB_CONTEXT
{}
- "integration":
<<: *DOCKERHUB_CONTEXT
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
- "debian-9"
- "typechecks":
<<: *DOCKERHUB_CONTEXT
{}
- "docs":
{}
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
@ -102,8 +87,16 @@ workflows:
- "master"
jobs:
- "build-image-debian-10":
<<: *DOCKERHUB_CONTEXT
# Every job that pushes a Docker image from Docker Hub needs to provide
# credentials. Use this first job to define a yaml anchor that can be
# used to supply a CircleCI job context which makes Docker Hub
# credentials available in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "build-image-debian-10": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "build-image-debian-9":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-16-04":
@ -458,6 +451,18 @@ jobs:
command: |
/tmp/venv/bin/tox -e typechecks
docs:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3"
steps:
- "checkout"
- run:
name: "Build documentation"
command: |
/tmp/venv/bin/tox -e docs
build-image: &BUILD_IMAGE
# This is a template for a job to build a Docker image that has as much of
# the setup as we can manage already done and baked in. This cuts down on

View File

@ -6,6 +6,10 @@ on:
- "master"
pull_request:
env:
# Tell Hypothesis which configuration we want it to use.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
jobs:
coverage:
@ -14,22 +18,22 @@ jobs:
fail-fast: false
matrix:
os:
- macos-latest
- windows-latest
- ubuntu-latest
python-version:
- 2.7
- 3.6
- 3.7
- 3.8
- 3.9
include:
# On macOS don't bother with 3.6-3.8, just to get faster builds.
- os: macos-latest
python-version: 2.7
- os: macos-latest
python-version: 3.9
steps:
# Get vcpython27 on Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
uses: crazy-max/ghaction-chocolatey@v1
with:
args: install vcpython27
# See https://github.com/actions/checkout. A fetch-depth of 0
# fetches all tags and branches.
- name: Check out Tahoe-LAFS sources
@ -38,7 +42,7 @@ jobs:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
@ -63,14 +67,14 @@ jobs:
- name: Install Python packages
run: |
pip install --upgrade codecov tox setuptools
pip install --upgrade codecov tox tox-gh-actions setuptools
pip list
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run "tox -e py27-coverage"
run: tox -e py27-coverage
- name: Run tox for corresponding Python version
run: python -m tox
- name: Upload eliot.log in case of failure
uses: actions/upload-artifact@v1
@ -83,10 +87,29 @@ jobs:
# Action for this, as of Jan 2021 it does not support Python coverage
# files - only lcov files. Therefore, we use coveralls-python, the
# coveralls.io-supplied Python reporter, for this.
#
# It is coveralls-python 1.x that has maintained compatibility
# with Python 2, while coveralls-python 3.x is compatible with
# Python 3. Sadly we can't use them both in the same workflow.
#
# The two versions of coveralls-python are somewhat mutually
# incompatible. Mixing these two different versions when
# reporting coverage to coveralls.io will lead to grief, since
# they get job IDs in different fashion. If we use both
# versions of coveralls in the same workflow, the finalizing
# step will be able to mark only part of the jobs as done, and
# the other part will be left hanging, never marked as done: it
# does not matter if we make an API call or `coveralls --finish`
# to indicate that CI has finished running.
#
# So we try to use the newer coveralls-python that is available
# via Python 3 (which is present in GitHub Actions tool cache,
# even when we're running Python 2.7 tests) throughout this
# workflow.
- name: "Report Coverage to Coveralls"
run: |
pip install coveralls
python -m coveralls
pip3 install --upgrade coveralls==3.0.1
python3 -m coveralls
env:
# Some magic value required for some magic reason.
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@ -109,80 +132,22 @@ jobs:
# a single report, we have to tell Coveralls when we've uploaded all of the
# data files. This does it. We make sure it runs last by making it depend
# on *all* of the coverage-collecting jobs.
#
# See notes about parallel builds on GitHub Actions at
# https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html
finish-coverage-report:
# There happens to just be one coverage-collecting job at the moment. If
# the coverage reports are broken and someone added more
# coverage-collecting jobs to this workflow but didn't update this, that's
# why.
needs:
- "coverage"
runs-on: "ubuntu-latest"
container: "python:3-slim"
steps:
- name: "Check out Tahoe-LAFS sources"
uses: "actions/checkout@v2"
- name: "Finish Coveralls Reporting"
- name: "Indicate completion to coveralls.io"
run: |
# coveralls-python does have a `--finish` option but it doesn't seem
# to work, at least for us.
# https://github.com/coveralls-clients/coveralls-python/issues/248
#
# But all it does is this simple POST so we can just send it
# ourselves. The only hard part is guessing what the POST
# parameters mean. And I've done that for you already.
#
# Since the build is done I'm going to guess that "done" is a fine
# value for status.
#
# That leaves "build_num". The coveralls documentation gives some
# hints about it. It suggests using $CIRCLE_WORKFLOW_ID if your job
# is on CircleCI. CircleCI documentation says this about
# CIRCLE_WORKFLOW_ID:
#
# Observation of the coveralls.io web interface, logs from the
# coveralls command in action, and experimentation suggests the
# value for PRs is something more like:
#
# <GIT MERGE COMMIT HASH>-PR-<PR NUM>
#
# For branches, it's just the git branch tip hash.
# For pull requests, refs/pull/<PR NUM>/merge was just checked out
# by so HEAD will refer to the right revision. For branches, HEAD
# is also the tip of the branch.
REV=$(git rev-parse HEAD)
# We can get the PR number from the "context".
#
# https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request
#
# (via <https://github.community/t/github-ref-is-inconsistent/17728/3>).
#
# If this is a pull request, `github.event` is a `pull_request`
# structure which has `number` right in it.
#
# If this is a push, `github.event` is a `push` instead but we only
# need the revision to construct the build_num.
PR=${{ github.event.number }}
if [ "${PR}" = "" ]; then
BUILD_NUM=$REV
else
BUILD_NUM=$REV-PR-$PR
fi
REPO_NAME=$GITHUB_REPOSITORY
curl \
-k \
https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \
-d \
"payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME"
pip3 install --upgrade coveralls==3.0.1
python3 -m coveralls --finish
env:
# Some magic value required for some magic reason.
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
# Help coveralls identify our project.
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
integration:
runs-on: ${{ matrix.os }}
@ -190,29 +155,34 @@ jobs:
fail-fast: false
matrix:
os:
- macos-latest
- windows-latest
- ubuntu-latest
python-version:
- 2.7
- 3.6
- 3.9
include:
# On macOS don't bother with 3.6, just to get faster builds.
- os: macos-latest
python-version: 2.7
- os: macos-latest
python-version: 3.9
steps:
# Get vcpython27 for Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
uses: crazy-max/ghaction-chocolatey@v1
with:
args: install vcpython27
- name: Install Tor [Ubuntu]
if: matrix.os == 'ubuntu-latest'
run: sudo apt install tor
- name: Install Tor [macOS]
if: matrix.os == 'macos-latest'
run: brew install tor
# TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744.
# We have to use an older version of Tor for running integration
# tests on macOS.
- name: Install Tor [macOS, ${{ matrix.python-version }} ]
if: ${{ matrix.os == 'macos-latest' }}
run: |
brew extract --version 0.4.5.8 tor homebrew/cask
brew install tor@0.4.5.8
brew link --overwrite tor@0.4.5.8
- name: Install Tor [Windows]
if: matrix.os == 'windows-latest'
@ -251,9 +221,14 @@ jobs:
- name: Display tool versions
run: python misc/build_helpers/show-tool-versions.py
- name: Run "tox -e integration"
- name: Run "Python 2 integration tests"
if: ${{ matrix.python-version == '2.7' }}
run: tox -e integration
- name: Run "Python 3 integration tests"
if: ${{ matrix.python-version != '2.7' }}
run: tox -e integration3
- name: Upload eliot.log in case of failure
uses: actions/upload-artifact@v1
if: failure()
@ -275,15 +250,6 @@ jobs:
steps:
# Get vcpython27 for Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
uses: crazy-max/ghaction-chocolatey@v1
with:
args: install vcpython27
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
with:

View File

@ -1,22 +0,0 @@
extraction:
python:
after_prepare:
- |
# https://discuss.lgtm.com/t/determination-of-python-requirements/974/4
sed -i 's/\("pyOpenSSL\)/\# Dependency removed for lgtm (see .lgtm.yml): \1/g' src/allmydata/_auto_deps.py
queries:
# This generates spurious errors for calls by interface because of the
# zope.interface choice to exclude self from method signatures. So, turn it
# off.
- exclude: "py/call/wrong-arguments"
# The premise of this query is broken. The errors it produces are nonsense.
# There is no such thing as a "procedure" in Python and "None" is not
# meaningless.
- exclude: "py/procedure-return-value-used"
# It is true that this query identifies things which are sometimes mistakes.
# However, it also identifies things which are entirely valid. Therefore,
# it produces noisy results.
- exclude: "py/implicit-string-concatenation-in-list"

42
CONTRIBUTORS.rst Normal file
View File

@ -0,0 +1,42 @@
Contributor Checklist
=====================
* Create a ``Trac`` ticket, fill it out and assign it to yourself (contact exarkun if you don't have an account):
``https://tahoe-lafs.org/trac/tahoe-lafs/newticket``
* Use the ticket number to name your branch (example):
``3003.contributor-guide``
* Good idea to add tests at the same time you write your code.
* Add a file to the ``/newsfragments`` folder, named with the ticket number and the type of patch (example):
``newsfragments/3651.minor``
* ``towncrier`` recognizes the following types:
``incompat``, ``feature``, ``bugfix``, ``installation``, ``configuration``, ``documentation``, ``removed``, ``other``, ``minor``
* Add one sentence to ``newsfragments/<ticket-number>.<towncrier-type>`` describing the change (example):
``The integration test suite has been updated to use pytest-twisted instead of deprecated pytest APIs.``
* Run the test suite with ``tox``, ``tox -e codechecks`` and ``tox -e typechecks``
* Push your branch to Github with your ticket number in the merge commit message (example):
``Fixes ticket:3003``
This makes the ``Trac`` ticket close when your PR gets approved.
* Request appropriate review - we suggest asking `Tahoe Committers <https://github.com/orgs/tahoe-lafs/teams/tahoe-committers>`__
References
----------
This checklist is a summary of `this page on contributing Patches <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Patches>`__
Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`__
and the `Contributor Code of Conduct <docs/CODE_OF_CONDUCT.md>`__.

28
CREDITS
View File

@ -204,6 +204,34 @@ E: meejah@meejah.ca
P: 0xC2602803128069A7, 9D5A 2BD5 688E CB88 9DEB CD3F C260 2803 1280 69A7
D: various bug-fixes and features
N: Chad Whitacre
E: chad@zetaweb.com
D: Python3 porting
N: Itamar Turner-Trauring
E: itamar@pythonspeed.com
D: Python3 porting
N: Jason R. Coombs
E: jaraco@jaraco.com
D: Python3 porting
N: Maciej Fijalkowski
E: fijall@gmail.com
D: Python3 porting
N: Ross Patterson
E: me@rpatterson.net
D: Python3 porting
N: Sajith Sasidharan
E: sajith@hcoop.net
D: Python3 porting
N: Pete Fein
E: pete@snake.dev
D: Python3 porting
N: Viktoriia Savchuk
W: https://twitter.com/viktoriiasvchk
D: Developer community focused improvements on the README file.

File diff suppressed because one or more lines are too long

View File

@ -6,7 +6,7 @@ Free and Open decentralized data store
`Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
|Contributor Covenant| |readthedocs| |travis| |circleci| |coveralls|
|Contributor Covenant| |readthedocs| |circleci| |githubactions| |coveralls|
Table of contents
@ -72,7 +72,7 @@ You can find the full Tahoe-LAFS documentation at our `documentation site <http:
Get involved with the Tahoe-LAFS community:
- Chat with Tahoe-LAFS developers at #tahoe-lafs chat on irc.freenode.net or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
- Chat with Tahoe-LAFS developers at ``#tahoe-lafs`` channel on `libera.chat <https://libera.chat/>`__ IRC network or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
- Join our `weekly conference calls <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/WeeklyMeeting>`__ with core developers and interested community members.
@ -93,6 +93,10 @@ As a community-driven open source project, Tahoe-LAFS welcomes contributions of
Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`__ and the `Contributor Code of Conduct <docs/CODE_OF_CONDUCT.md>`__.
🤝 Supporters
--------------
We would like to thank `Fosshost <https://fosshost.org>`__ for supporting us with hosting services. If your open source project needs help, you can apply for their support.
❓ FAQ
------
@ -118,13 +122,12 @@ See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL ex
:alt: documentation status
:target: http://tahoe-lafs.readthedocs.io/en/latest/?badge=latest
.. |travis| image:: https://travis-ci.org/tahoe-lafs/tahoe-lafs.png?branch=master
:alt: build status
:target: https://travis-ci.org/tahoe-lafs/tahoe-lafs
.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
:target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
.. |githubactions| image:: https://github.com/tahoe-lafs/tahoe-lafs/actions/workflows/ci.yml/badge.svg
:target: https://github.com/tahoe-lafs/tahoe-lafs/actions
.. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg
:alt: code coverage
:target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs

View File

@ -45,6 +45,7 @@ The following community members have made themselves available for conduct issue
- Jean-Paul Calderone (jean-paul at leastauthority dot com)
- meejah (meejah at meejah dot ca)
- May-Lee Sia(she/her) (tahoe dot lafs dot community at gmail dot com)
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
version 1.3.0, available at

View File

@ -1,343 +0,0 @@
.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*-
..
note: if you aren't reading the rendered form of these docs at
http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any
":doc:" links refer to other files in this docs/ directory
*********************
Installing Tahoe-LAFS
*********************
Welcome to `the Tahoe-LAFS project`_, a secure, decentralized, fault-tolerant
storage system. See :doc:`about` for an overview of the architecture and
security properties of the system.
This procedure should work on Windows, Mac, illumos (previously OpenSolaris),
and too many flavors of Linux and of BSD to list.
.. _the Tahoe-LAFS project: https://tahoe-lafs.org
First: In Case Of Trouble
=========================
In some cases these instructions may fail due to peculiarities of your
platform.
If the following instructions don't Just Work without any further effort on
your part, then please write to `the tahoe-dev mailing list`_ where friendly
hackers will help you out.
.. _the tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev
Pre-Packaged Versions
=====================
You may not need to build Tahoe at all.
If you are on Windows, please see :doc:`windows` for platform-specific
instructions.
If you are on a Mac, you can either follow these instructions, or use the
pre-packaged bundle described in :doc:`OS-X`.
Many Linux distributions include Tahoe-LAFS packages. Debian and Ubuntu users
can ``apt-get install tahoe-lafs``. See `OSPackages`_ for other
platforms.
.. _OSPackages: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/OSPackages
Preliminaries
=============
If you don't use a pre-packaged copy of Tahoe, you can build it yourself.
You'll need Python2.7, pip, and virtualenv.
Tahoe-LAFS depends on some libraries which require a C compiler to build.
However, for many platforms, PyPI hosts already-built packages of libraries.
If there is no already-built package for your platform,
you will need a C compiler,
the Python development headers,
and some libraries (libffi-dev and libssl-dev).
On a modern Debian/Ubuntu-derived distribution, this command will get you
everything you need::
apt-get install build-essential python-dev libffi-dev libssl-dev libyaml-dev python-virtualenv
On OS-X, install pip and virtualenv as described below. If you want to
compile the dependencies yourself, you'll also need to install
Xcode and its command-line tools.
**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
Python 2.7
----------
Check if you already have an adequate version of Python installed by running
``python -V``. The latest version of Python v2.7 is recommended, which is
2.7.11 as of this writing. Python v2.6.x and v3 do not work. On Windows, we
recommend the use of native Python v2.7, not Cygwin Python. If you don't have
one of these versions of Python installed, `download`_ and install the latest
version of Python v2.7. Make sure that the path to the installation directory
has no spaces in it (e.g. on Windows, do not install Python in the "Program
Files" directory)::
% python --version
Python 2.7.11
.. _download: https://www.python.org/downloads/
pip
---
Many Python installations already include ``pip``, but in case yours does
not, get it with the `pip install instructions`_::
% pip --version
pip 10.0.1 from ... (python 2.7)
.. _pip install instructions: https://pip.pypa.io/en/stable/installing/
virtualenv
----------
If you do not have an OS-provided copy of ``virtualenv``, install it with the
instructions from the `virtualenv documentation`_::
% virtualenv --version
15.1.0
.. _virtualenv documentation: https://virtualenv.pypa.io/en/latest/installation.html
C compiler and libraries
------------------------
Except on OS-X, where the Tahoe project hosts pre-compiled wheels for all
dependencies, you will need several C libraries installed before you can
build. You will also need the Python development headers, and a C compiler
(your python installation should know how to find these).
On Debian/Ubuntu-derived systems, the necessary packages are ``python-dev``,
``libffi-dev``, and ``libssl-dev``, and can be installed with ``apt-get``. On
RPM-based system (like Fedora) these may be named ``python-devel``, etc,
instead, and cam be installed with ``yum`` or ``rpm``.
**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
Install the Latest Tahoe-LAFS Release
=====================================
We recommend creating a fresh virtualenv for your Tahoe-LAFS install, to
isolate it from any python packages that are already installed (and to
isolate the rest of your system from Tahoe's dependencies).
This example uses a virtualenv named ``venv``, but you can call it anything
you like. Many people prefer to keep all their virtualenvs in one place, like
``~/.local/venvs/`` or ``~/venvs/``.
It's usually a good idea to upgrade the virtualenv's ``pip`` and
``setuptools`` to their latest versions, with ``venv/bin/pip install -U pip
setuptools``. Many operating systems have an older version of ``virtualenv``,
which then includes older versions of pip and setuptools. Upgrading is easy,
and only affects the virtualenv: not the rest of your computer.
Then use the virtualenv's ``pip`` to install the latest Tahoe-LAFS release
from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run
``venv/bin/tahoe --version`` to confirm the install was successful::
% virtualenv venv
New python executable in ~/venv/bin/python2.7
Installing setuptools, pip, wheel...done.
% venv/bin/pip install -U pip setuptools
Downloading/unpacking pip from https://pypi.python.org/...
...
Successfully installed pip setuptools
% venv/bin/pip install tahoe-lafs
Collecting tahoe-lafs
...
Installing collected packages: ...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.14.0
foolscap: ...
%
Install From a Source Tarball
-----------------------------
You can also install directly from the source tarball URL. To verify
signatures, first see verifying_signatures_ and replace the URL in the
following instructions with the local filename.
% virtualenv venv
New python executable in ~/venv/bin/python2.7
Installing setuptools, pip, wheel...done.
% venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2
Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.14.0.tar.bz2
...
Installing collected packages: ...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.14.0
...
.. _verifying_signatures:
Verifying Signatures
--------------------
First download the source tarball and then any signatures. There are several
developers who are able to produce signatures for a release. A release may
have multiple signatures. All should be valid and you should confirm at least
one of them (ideally, confirm all).
This statement, signed by the existing Tahoe release-signing key, attests to
those developers authorized to sign a Tahoe release:
.. include:: developer-release-signatures
:code:
Signatures are made available beside the release. So for example, a release
like ``https://tahoe-lafs.org/downloads/tahoe-lafs-1.16.0.tar.bz2`` might
have signatures ``tahoe-lafs-1.16.0.tar.bz2.meejah.asc`` and
``tahoe-lafs-1.16.0.tar.bz2.warner.asc``.
To verify the signatures using GnuPG::
% gpg --verify tahoe-lafs-1.16.0.tar.bz2.meejah.asc tahoe-lafs-1.16.0.tar.bz2
gpg: Signature made XXX
gpg: using RSA key 9D5A2BD5688ECB889DEBCD3FC2602803128069A7
gpg: Good signature from "meejah <meejah@meejah.ca>" [full]
% gpg --verify tahoe-lafs-1.16.0.tar.bz2.warner.asc tahoe-lafs-1.16.0.tar.bz2
gpg: Signature made XXX
gpg: using RSA key 967EFE06699872411A77DF36D43B4C9C73225AAF
gpg: Good signature from "Brian Warner <warner@lothar.com>" [full]
Extras
------
Tahoe-LAFS provides some functionality only when explicitly requested at installation time.
It does this using the "extras" feature of setuptools.
You can request these extra features when running the ``pip install`` command like this::
% venv/bin/pip install tahoe-lafs[tor]
This example enables support for listening and connecting using Tor.
The Tahoe-LAFS documentation for specific features which require an explicit install-time step will mention the "extra" that must be requested.
Hacking On Tahoe-LAFS
---------------------
To modify the Tahoe source code, you should get a git checkout, and install
with the ``--editable`` flag. You should also use the ``[test]`` extra to get
the additional libraries needed to run the unit tests::
% git clone https://github.com/tahoe-lafs/tahoe-lafs.git
% cd tahoe-lafs
% virtualenv venv
% venv/bin/pip install --editable .[test]
Obtaining file::~/tahoe-lafs
...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.14.0.post34.dev0
...
This way, you won't have to re-run the ``pip install`` step each time you
modify the source code.
Running the ``tahoe`` executable
================================
The rest of the Tahoe-LAFS documentation assumes that you can run the
``tahoe`` executable that you just created. You have four basic options:
* Use the full path each time (e.g. ``~/venv/bin/tahoe``).
* "`Activate`_" the virtualenv with ``. venv/bin/activate``, to get a
subshell with a ``$PATH`` that includes the ``venv/bin/`` directory, then
you can just run ``tahoe``.
* Change your ``$PATH`` to include the ``venv/bin/`` directory, so you can
just run ``tahoe``.
* Symlink from ``~/bin/tahoe`` to the ``tahoe`` executable. Since ``~/bin``
is typically in your ``$PATH`` (at least if it exists when you log in),
this will let you just run ``tahoe``.
You might also find the `pipsi`_ tool convenient: ``pipsi install
tahoe-lafs`` will create a new virtualenv, install tahoe into it, then
symlink just the executable (into ``~/.local/bin/tahoe``). Then either add
``~/.local/bin/`` to your ``$PATH``, or make one last symlink into
``~/bin/tahoe``.
.. _Activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script
.. _pipsi: https://pypi.python.org/pypi/pipsi/0.9
Running the Self-Tests
======================
To run the self-tests from a source tree, you'll need ``tox`` installed. On a
Debian/Ubuntu system, use ``apt-get install tox``. You can also install it
into your tahoe-specific virtualenv with ``pip install tox``.
Then just run ``tox``. This will create a new fresh virtualenv, install Tahoe
(from the source tree, including any changes you have made) and all its
dependencies (including testing-only dependencies) into the virtualenv, then
run the unit tests. This ensures that the tests are repeatable and match the
results of other users, unaffected by any other Python packages installed on
your machine. On a modern computer this will take 5-10 minutes, and should
result in a "all tests passed" mesage::
% tox
GLOB sdist-make: ~/tahoe-lafs/setup.py
py27 recreate: ~/tahoe-lafs/.tox/py27
py27 inst: ~/tahoe-lafs/.tox/dist/tahoe-lafs-1.14.0.post8.dev0.zip
py27 runtests: commands[0] | tahoe --version
py27 runtests: commands[1] | trial --rterrors allmydata
allmydata.test.test_auth
AccountFileCheckerKeyTests
test_authenticated ... [OK]
test_missing_signature ... [OK]
...
Ran 1186 tests in 423.179s
PASSED (skips=7, expectedFailures=3, successes=1176)
__________________________ summary ___________________________________
py27: commands succeeded
congratulations :)
Common Problems
===============
If you see an error like ``fatal error: Python.h: No such file or directory``
while compiling the dependencies, you need the Python development headers. If
you are on a Debian or Ubuntu system, you can install them with ``sudo
apt-get install python-dev``. On RedHat/Fedora, install ``python-devel``.
Similar errors about ``openssl/crypto.h`` indicate that you are missing the
OpenSSL development headers (``libssl-dev``). Likewise ``ffi.h`` means you
need ``libffi-dev``.
**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
Using Tahoe-LAFS
================
Now you are ready to deploy a decentralized filesystem. You will use the
``tahoe`` executable to create, configure, and launch your Tahoe-LAFS nodes.
See :doc:`running` for instructions on how to do that.

View File

@ -1,6 +1,6 @@
******************************************
How To Build Tahoe-LAFS On A Desert Island
******************************************
***************************************
Building Tahoe-LAFS On A Desert Island
***************************************
(or an airplane, or anywhere else without internet connectivity)

View File

@ -0,0 +1,75 @@
****************************
Building Tahoe-LAFS on Linux
****************************
Tahoe-LAFS has made packages available for installing on many linux and BSD distributions.
Debian and Ubuntu users can use ``apt-get install tahoe-lafs``.
If you are working on a Linux distribution which does not have Tahoe-LAFS or are looking to hack on the source code, you can build Tahoe-LAFS yourself:
Prerequisites
=============
Make sure the following are installed:
* **Python 3's latest version**: Check for the version by running ``python --version``.
* **pip**: Most python installations already include ``pip``. However, if your installation does not, see `pip installation <https://pip.pypa.io/en/stable/installing/>`_.
* **virtualenv**: Use ``pip`` to install virtualenv::
pip install --user virtualenv
* **C compiler and libraries**:
* ``python-dev``: Python development headers.
* ``libffi-dev``: Foreign Functions Interface library.
* ``libssl-dev``: SSL library, Tahoe-LAFS needs OpenSSL version 1.1.1c or greater.
.. note::
If you are working on Debian or Ubuntu, you can install the necessary libraries using ``apt-get``::
apt-get install python-dev libffi-dev libssl-dev
On an RPM-based system such as Fedora, you can install the necessary libraries using ``yum`` or ``rpm``. However, the packages may be named differently.
Install the Latest Tahoe-LAFS Release
=====================================
If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS directly from source by creating a ``virtualenv`` instance:
1. Clone the Tahoe-LAFS repository::
git clone https://github.com/tahoe-lafs/tahoe-lafs.git
2. Move into the tahoe-lafs directory::
cd tahoe-lafs
3. Create a fresh virtualenv for your Tahoe-LAFS install::
virtualenv venv
.. note::
venv is the name of the virtual environment in this example. Use any name for your environment.
4. Upgrade ``pip`` and ``setuptools`` on the newly created virtual environment::
venv/bin/pip install -U pip setuptools
5. If you'd like to modify the Tahoe source code, you need to install Tahoe-LAFS with the ``--editable`` flag with the ``test`` extra::
venv/bin/pip install --editable .[test]
.. note::
Tahoe-LAFS provides extra functionality when requested explicitly at installation using the "extras" feature of setuptools. To learn more about the extras which Tahoe supports, see Tahoe extras.
6. Verify installation by checking for the version::
venv/bin/tahoe --version
If you do not want to use the full path, i.e., ``venv/bin/tahoe`` everytime you want to run tahoe, you can activate the ``virtualenv``::
. venv/bin/activate
This will generate a subshell with a ``$PATH`` that includes the ``venv/bin/`` directory.

View File

@ -0,0 +1,45 @@
******************************
Building Tahoe-LAFS on Windows
******************************
If you are looking to hack on the source code or run pre-release code, we recommend you create a virtualenv instance and install Tahoe-LAFS into that:
1. Make sure you have Powershell installed. See `PowerShell installation <https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-7.1>`_.
2. Install the latest version of Python 3. Download the .exe file at the `python website <https://www.python.org/downloads/>`_.
3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**.
4. Start PowerShell and enter the following command to verify python installation::
python --version
5. Use ``pip`` to install ``virtualenv``::
pip install --user virtualenv
6. Create a fresh virtualenv for your Tahoe-LAFS install using the following command::
virtualenv venv
.. note::
venv is the name of the virtual environment in this example. Use any name for your environment.
7. Use pip to install Tahoe-LAFS in the virtualenv instance::
venv\Scripts\pip install tahoe-lafs
6. Verify installation by checking for the version::
venv\Scripts\tahoe --version
If you do not want to use the full path, i.e. ``venv\Scripts\tahoe`` everytime you want to run tahoe, you can:
* Activate the virtualenv::
. venv\Scripts\activate
This will generate a subshell with a ``$PATH`` that includes the ``venv\Scripts\`` directory.
* Change your ``$PATH`` to include the ``venv\Scripts`` directory.

View File

@ -0,0 +1,68 @@
.. -*- coding: utf-8-with-signature-unix; fill-column: 77 -*-
..
note: if you aren't reading the rendered form of these docs at
http://tahoe-lafs.readthedocs.io/en/latest/ , then be aware that any
":doc:" links refer to other files in this docs/ directory
*********************
Installing Tahoe-LAFS
*********************
`Tahoe-LAFS`_ is a secure, decentralized, and fault-tolerant storage system.
To see an overview of the architecture and security properties, see :doc:`Welcome to Tahoe LAFS! <../about-tahoe>`
Tahoe-LAFS can be installed and used on any of the following operating systems.
.. _Tahoe-LAFS: https://tahoe-lafs.org
Microsoft Windows
=================
To install Tahoe-LAFS on Windows:
1. Make sure you have Powershell installed. See `PowerShell installation <https://docs.microsoft.com/en-us/powershell/scripting/install/installing-powershell-core-on-windows?view=powershell-7.1>`_.
2. Install the latest version of Python 3. Download the .exe file at the `python website <https://www.python.org/downloads/>`_.
3. Open the installer by double-clicking it. Select the **Add Python to PATH** check-box, then click **Install Now**.
4. Start PowerShell and enter the following command to verify python installation::
python --version
5. Enter the following command to install Tahoe-LAFS::
pip install tahoe-lafs
6. Verify installation by checking for the version::
tahoe --version
If you want to hack on Tahoe's source code, you can install Tahoe in a ``virtualenv`` on your Windows Machine. To learn more, see :doc:`install-on-windows`.
Linux, BSD, or MacOS
====================
Tahoe-LAFS can be installed on MacOS, many Linux and BSD distributions. If you are using Ubuntu or Debian, run the following command to install Tahoe-LAFS::
apt-get install tahoe-lafs
If you are working on MacOS or a Linux distribution which does not have Tahoe-LAFS packages, you can build it yourself:
1. Make sure the following are installed:
* **Python 3's latest version**: Check for the version by running ``python --version``.
* **pip**: Most python installations already include `pip`. However, if your installation does not, see `pip installation <https://pip.pypa.io/en/stable/installing/>`_.
2. Install Tahoe-LAFS using pip::
pip install tahoe-lafs
3. Verify installation by checking for the version::
tahoe --version
If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`.
You can always write to the `tahoe-dev mailing list <https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev>`_ or chat on the `Libera.chat IRC <irc://irc.libera.chat/%23tahoe-lafs>`_ if you are not able to get Tahoe-LAFS up and running on your deployment.

View File

@ -214,3 +214,7 @@ pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
.PHONY: livehtml
livehtml:
sphinx-autobuild -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html

View File

@ -1,23 +0,0 @@
==============
OS-X Packaging
==============
Pre-built Tahoe-LAFS ".pkg" installers for OS-X are generated with each
source-code commit. These installers offer an easy way to get Tahoe and all
its dependencies installed on your Mac. They do not yet provide a
double-clickable application: after installation, you will have a "tahoe"
command-line tool, which you can use from a shell (a Terminal window) just as
if you'd installed from source.
Installers are available from this directory:
https://tahoe-lafs.org/source/tahoe-lafs/tarballs/OS-X-packages/
Download the latest .pkg file to your computer and double-click on it. This
will install to /Applications/tahoe.app, however the app icon there is not
how you use Tahoe (launching it will get you a dialog box with a reminder to
use Terminal). ``/Applications/tahoe.app/bin/tahoe`` is the executable. The
next shell you start ought to have that directory in your $PATH (thanks to a
file in ``/etc/paths.d/``), unless your ``.profile`` overrides it.
Tahoe-LAFS is also easy to install with pip, as described in the README.

View File

@ -1,7 +1,30 @@
If you are reading Tahoe-LAFS documentation
-------------------------------------------
Note: http://tahoe-lafs.readthedocs.io/en/latest/ is the preferred place to
read this documentation (GitHub doesn't render cross-document links or
images). If you're reading this on https://github.com/tahoe-lafs/tahoe-lafs ,
or from a checked-out source tree, then either run `tox -e docs` and open
_build/html/index.html in your browser, or view the pre-rendered trunk copy
at http://tahoe-lafs.readthedocs.io/en/latest/
If you are reading Tahoe-LAFS documentation at a code hosting site or
from a checked-out source tree, the preferred place to view the docs
is http://tahoe-lafs.readthedocs.io/en/latest/. Code-hosting sites do
not render cross-document links or images correctly.
If you are writing Tahoe-LAFS documentation
-------------------------------------------
To edit Tahoe-LAFS docs, you will need a checked-out source tree. You
can edit the `.rst` files in this directory using a text editor, and
then generate HTML output using Sphinx, a program that can produce its
output in HTML and other formats.
Files with `.rst` extension use reStructuredText markup format, which
is the format Sphinx natively handles. To learn more about Sphinx, and
for a friendly primer on reStructuredText, please see Sphinx project's
documentation, available at:
https://www.sphinx-doc.org/
If you have `tox` installed, you can run `tox -e docs` and then open
the resulting docs/_build/html/index.html in your web browser.
Note that Sphinx can also process Python docstrings to generate API
documentation. Tahoe-LAFS currently does not use Sphinx for this
purpose.

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.3 KiB

After

Width:  |  Height:  |  Size: 7.6 KiB

View File

@ -127,7 +127,7 @@ For more technical detail, please see the `the doc page`_ on the Wiki.
Get Started
===========
To use Tahoe-LAFS, please see :doc:`INSTALL`.
To use Tahoe-LAFS, please see :doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`.
License
=======

View File

@ -28,7 +28,7 @@ import os
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['recommonmark']
extensions = ['recommonmark', 'sphinx_rtd_theme']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@ -107,7 +107,7 @@ todo_include_todos = False
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'default'
html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the

View File

@ -514,10 +514,10 @@ Command Examples
the pattern will be matched against any level of the directory tree;
it's still impossible to specify absolute path exclusions.
``tahoe backup --exclude-from=/path/to/filename ~ work:backups``
``tahoe backup --exclude-from-utf-8=/path/to/filename ~ work:backups``
``--exclude-from`` is similar to ``--exclude``, but reads exclusion
patterns from ``/path/to/filename``, one per line.
``--exclude-from-utf-8`` is similar to ``--exclude``, but reads exclusion
patterns from a UTF-8-encoded ``/path/to/filename``, one per line.
``tahoe backup --exclude-vcs ~ work:backups``

View File

@ -7,11 +7,10 @@ Tahoe-LAFS SFTP Frontend
1. `SFTP Background`_
2. `Tahoe-LAFS Support`_
3. `Creating an Account File`_
4. `Running An Account Server (accounts.url)`_
5. `Configuring SFTP Access`_
6. `Dependencies`_
7. `Immutable and Mutable Files`_
8. `Known Issues`_
4. `Configuring SFTP Access`_
5. `Dependencies`_
6. `Immutable and Mutable Files`_
7. `Known Issues`_
SFTP Background
@ -78,33 +77,6 @@ start with "ssh-".
Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in
the next sections.
Running An Account Server (accounts.url)
========================================
The accounts.url directive allows access requests to be controlled by an
HTTP-based login service, useful for centralized deployments. This was used
by AllMyData to provide web-based file access, where the service used a
simple PHP script and database lookups to map an account email address and
password to a Tahoe-LAFS directory cap. The service will receive a
multipart/form-data POST, just like one created with a <form> and <input>
fields, with three parameters:
• action: "authenticate" (this is a static string)
• email: USERNAME (Tahoe-LAFS has no notion of email addresses, but the
authentication service uses them as account names, so the interface
presents this argument as "email" rather than "username").
• passwd: PASSWORD
It should return a single string that either contains a Tahoe-LAFS directory
cap (URI:DIR2:...), or "0" to indicate a login failure.
Tahoe-LAFS recommends the service be secure, preferably localhost-only. This
makes it harder for attackers to brute force the password or use DNS
poisoning to cause the Tahoe-LAFS gateway to talk with the wrong server,
thereby revealing the usernames and passwords.
Public key authentication is not supported when an account server is used.
Configuring SFTP Access
=======================

View File

@ -10,8 +10,11 @@ Contents:
.. toctree::
:maxdepth: 2
about
INSTALL
about-tahoe
Installation/install-tahoe
Installation/install-on-windows
Installation/install-on-linux
Installation/install-on-desert-island
running
magic-wormhole-invites
configuration
@ -41,6 +44,7 @@ Contents:
backupdb
developer-guide
ticket-triage
anonymity-configuration
@ -49,10 +53,7 @@ Contents:
logging
stats
desert-island
debian
windows
OS-X
build/build-pyOpenSSL
specifications/index

View File

@ -13,6 +13,61 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit
The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change
(though this may be the first step towards simplifying it).
Motivation
----------
Foolscap
~~~~~~~~
Foolscap is a remote method invocation protocol with several distinctive features.
At its core it allows separate processes to refer each other's objects and methods using a capability-based model.
This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated.
Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways.
Tahoe-LAFS avails itself of only a small portion of these features.
A Tahoe-LAFS storage server typically only exposes one object with a fixed set of methods to clients.
A Tahoe-LAFS introducer node does roughly the same.
Tahoe-LAFS exchanges simple data structures that have many common, standard serialized representations.
In exchange for this slight use of Foolscap's sophisticated mechanisms,
Tahoe-LAFS pays a substantial price:
* Foolscap is implemented only for Python.
Tahoe-LAFS is thus limited to being implemented only in Python.
* There is only one Python implementation of Foolscap.
The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation.
* The Foolscap developer community is very small.
The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project.
* The extensible serialization system imposes substantial complexity compared to the simple data structures Tahoe-LAFS actually exchanges.
HTTP
~~~~
HTTP is a request/response protocol that has become the lingua franca of the internet.
Combined with the principles of Representational State Transfer (REST) it is widely employed to create, update, and delete data in collections on the internet.
HTTP itself provides only modest functionality in comparison to Foolscap.
However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on.
By adopting HTTP in place of Foolscap Tahoe-LAFS can realize the following concrete benefits:
* Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available.
This change paves the way for new Tahoe-LAFS implementations using tools better suited for certain situations
(mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc).
* The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use.
This change reduces the barrier to entry for developers to contribute improvements to Tahoe-LAFS's network interactions.
* For any given language there is very likely an HTTP implementation with a large and active developer community.
Tahoe-LAFS can therefore benefit from the large effort being put into making better libraries for using HTTP.
* One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency.
The alignment of this core feature with a core activity of Tahoe-LAFS of transferring bulk data means that a substantial barrier to improved Tahoe-LAFS runtime performance will be eliminated.
TLS
~~~
The Foolscap-based protocol provides *some* of Tahoe-LAFS's confidentiality, integrity, and authentication properties by leveraging TLS.
An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties.
Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation
(rather than the standard "web" rules for validation).
Requirements
------------
@ -101,12 +156,12 @@ Alice generates a key pair and secures it properly.
Alice generates a self-signed storage node certificate with the key pair.
Alice's storage node announces (to an introducer) a fURL containing (among other information) the SPKI hash.
Imagine the SPKI hash is ``i5xb...``.
This results in a fURL of ``pb://i5xb...@example.com:443/g3m5...#v=2`` [#]_.
This results in a fURL of ``pb://i5xb...@example.com:443/g3m5...#v=1``.
Bob creates a client node pointed at the same introducer.
Bob's client node receives the announcement from Alice's storage node
(indirected through the introducer).
Bob's client node recognizes the fURL as referring to an HTTP-dialect server due to the ``v=2`` fragment.
Bob's client node recognizes the fURL as referring to an HTTP-dialect server due to the ``v=1`` fragment.
Bob's client node can now perform a TLS handshake with a server at the address in the fURL location hints
(``example.com:443`` in this example).
Following the above described validation procedures,
@ -156,7 +211,7 @@ Such an announcement will resemble this::
{
"anonymous-storage-FURL": "pb://...", # The old key
"gbs-anonymous-storage-url": "pb://...#v=2" # The new key
"gbs-anonymous-storage-url": "pb://...#v=1" # The new key
}
The transition process will proceed in three stages:
@ -234,6 +289,19 @@ Because of the simple types used throughout
and the equivalence described in `RFC 7049`_
these examples should be representative regardless of which of these two encodings is chosen.
HTTP Design
~~~~~~~~~~~
The HTTP interface described here is informed by the ideas of REST
(Representational State Transfer).
For ``GET`` requests query parameters are preferred over values encoded in the request body.
For other requests query parameters are encoded into the message body.
Many branches of the resource tree are conceived as homogenous containers:
one branch contains all of the share data;
another branch contains all of the lease data;
etc.
General
~~~~~~~
@ -252,11 +320,70 @@ For example::
"delete-mutable-shares-with-zero-length-writev": true,
"fills-holes-with-zero-bytes": true,
"prevents-read-past-end-of-share-data": true,
"gbs-anonymous-storage-url": "pb://...#v=2"
"gbs-anonymous-storage-url": "pb://...#v=1"
},
"application-version": "1.13.0"
}
``PUT /v1/lease/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Create a new lease that applies to all shares for the given storage index.
The details of the lease are encoded in the request body.
For example::
{"renew-secret": "abcd", "cancel-secret": "efgh"}
If there are no shares for the given ``storage_index``
then do nothing and return ``NO CONTENT``.
If the ``renew-secret`` value matches an existing lease
then that lease will be renewed instead.
The lease expires after 31 days.
Discussion
``````````
We considered an alternative where ``renew-secret`` and ``cancel-secret`` are placed in query arguments on the request path.
We chose to put these values into the request body to make the URL simpler.
Several behaviors here are blindly copied from the Foolscap-based storage server protocol.
* There is a cancel secret but there is no API to use it to cancel a lease.
* The lease period is hard-coded at 31 days.
* There is no way to differentiate between success and an unknown **storage index**.
* There are separate **add** and **renew** lease APIs.
These are not necessarily ideal behaviors
but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols.
It is expected that some or all of these behaviors may change in a future revision of the HTTP-based protocol.
``POST /v1/lease/:storage_index``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Renew an existing lease for all shares for the given storage index.
The details of the lease are encoded in the request body.
For example::
{"renew-secret": "abcd"}
If there are no shares for the given ``storage_index``
then ``NOT FOUND`` is returned.
If there is no lease with a matching ``renew-secret`` value on the given storage index
then ``NOT FOUND`` is returned.
In this case,
if the storage index refers to mutable data
then the response also includes a list of nodeids where the lease can be renewed.
For example::
{"nodeids": ["aaa...", "bbb..."]}
Othewise,
the matching lease's expiration time is changed to be 31 days from the time of this operation
and ``NO CONTENT`` is returned.
Immutable
---------
@ -268,6 +395,7 @@ Writing
Initialize an immutable storage index with some buckets.
The buckets may have share data written to them once.
A lease is also created for the shares.
Details of the buckets to create are encoded in the request body.
For example::
@ -286,8 +414,13 @@ We considered making this ``POST /v1/immutable`` instead.
The motivation was to keep *storage index* out of the request URL.
Request URLs have an elevated chance of being logged by something.
We were concerned that having the *storage index* logged may increase some risks.
However, we decided this does not matter because the *storage index* can only be used to read the share (which is ciphertext).
TODO Verify this conclusion.
However, we decided this does not matter because:
* the *storage index* can only be used to retrieve (not decrypt) the ciphertext-bearing share.
* the *storage index* is already persistently present on the storage node in the form of directory names in the storage servers ``shares`` directory.
* the request is made via HTTPS and so only Tahoe-LAFS can see the contents,
therefore no proxy servers can perform any extra logging.
* Tahoe-LAFS itself does not currently log HTTP request URLs.
``PUT /v1/immutable/:storage_index/:share_number``
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@ -493,13 +626,6 @@ Just like the immutable version.
Note we use `base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.
.. [#]
Other schemes for differentiating between the two server types is possible.
If the tubID length remains different,
that provides an unambiguous (if obscure) signal about which protocol to use.
Or a different scheme could be adopted
(``[x-]pb+http``, ``x-tahoe+http``, ``x-gbs`` come to mind).
.. [#]
https://www.cvedetails.com/cve/CVE-2017-5638/
.. [#]

View File

@ -59,6 +59,10 @@ Create Branch and Apply Updates
- summarize major changes
- commit it
- update "nix/tahoe-lafs.nix"
- change the value given for `version` from `OLD.post1` to `NEW.post1`
- update "CREDITS"
- are there any new contributors in this release?
@ -66,7 +70,7 @@ Create Branch and Apply Updates
- commit it
- update "docs/known_issues.rst" if appropriate
- update "docs/INSTALL.rst" references to the new release
- update "docs/Installation/install-tahoe.rst" references to the new release
- Push the branch to github
- Create a (draft) PR; this should trigger CI (note that github
doesn't let you create a PR without some changes on the branch so
@ -189,11 +193,16 @@ is appropriate.
Once a release-candidate has marinated for some time then it can be
made into a the actual release.
XXX Write this section when doing 1.15.0 actual release
(In general, this means dropping the "rcX" part of the release and the
tag, uploading those artifacts, uploading to PyPI, ... )
The actual release follows the same steps as above, with some differences:
- there is no "-rcX" on the end of release names
- the release is uploaded to PyPI (using Twine)
- the version is tagged in Git (ideally using "the tahoe release key"
but can be done with any of the authorized core developers' personal
key)
- the release-candidate branches must be merged back to master after
the release is official (e.g. causing newsfragments to be deleted on
master, etc)
Announcing the Release

View File

@ -10,7 +10,7 @@ Introduction
This is how to run a Tahoe-LAFS client or a complete Tahoe-LAFS grid.
First you have to install the Tahoe-LAFS software, as documented in
:doc:`INSTALL`.
:doc:`Installing Tahoe-LAFS <../Installation/install-tahoe>`.
The ``tahoe`` program in your virtualenv's ``bin`` directory is used to
create, start, and stop nodes. Each node lives in a separate base
@ -235,7 +235,7 @@ Socialize
=========
You can chat with other users of and hackers of this software on the
#tahoe-lafs IRC channel at ``irc.freenode.net``, or on the `tahoe-dev mailing
#tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing
list`_.
.. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev

27
docs/ticket-triage.rst Normal file
View File

@ -0,0 +1,27 @@
=============
Ticket Triage
=============
Ticket triage is a weekly, informal ritual that is meant to solve the problem of
tickets getting opened and then forgotten about. It is simple and keeps project
momentum going and prevents ticket cruft.
It fosters conversation around project tasks and philosophies as they relate to
milestones.
Process
-------
- The role of Ticket Triager rotates regularly-ish, and is assigned ad hoc
- The Triager needs a ``Trac`` account
- The Triager looks at all the tickets that have been created in the last week (or month, etc.)
- They can use a custom query or do this as the week progresses
- BONUS ROUND: Dig up a stale ticket from the past
- Assign each ticket to a milestone on the Roadmap
- The following situations merit discussion:
- A ticket doesn't have an appropriate milestone and we should create one
- A ticket, in vanishingly rare circumstances, should be deleted
- The ticket is spam
- The ticket contains sensitive information and harm will come to one or more people if it continues to be distributed
- A ticket could be assigned to multiple milestones
- There is another question about a ticket
- These tickets will be brought as necessary to one of our meetings (currently Tuesdays) for discussion

View File

@ -1,83 +0,0 @@
Building Tahoe-LAFS on Windows
==============================
You'll need ``python``, ``pip``, and ``virtualenv``. But you won't need a
compiler.
Preliminaries
-------------
1: Install Python-2.7.11 . Use the "Windows x86-64 MSI installer" at
https://www.python.org/downloads/release/python-2711/
2: That should install ``pip``, but if it doesn't, look at
https://pip.pypa.io/en/stable/installing/ for installation instructions.
3: Install ``virtualenv`` with
https://virtualenv.pypa.io/en/latest/installation.html
Installation
------------
1: Start a CLI shell (e.g. PowerShell)
2: Create a new virtualenv. Everything specific to Tahoe will go into this.
You can use whatever name you like for the virtualenv, but example uses
"venv"::
PS C:\Users\me> virtualenv venv
New python executable in C:\Users\me\venv\Scripts\python.exe
Installing setuptools, pip, wheel...done.
>
3: Use the virtualenv's ``pip`` to install the latest release of Tahoe-LAFS
into this virtualenv::
PS C:\Users\me> venv\Scripts\pip install tahoe-lafs
Collecting tahoe-lafs
...
Installing collected packages: ...
Successfully installed ...
>
4: Verify that Tahoe was installed correctly by running ``tahoe --version``,
using the ``tahoe`` from the virtualenv's Scripts directory::
PS C:\Users\me> venv\Scripts\tahoe --version
tahoe-lafs: 1.11
foolscap: ...
Running Tahoe-LAFS
------------------
The rest of the documentation assumes you can run the ``tahoe`` executable
just as you did in step 4 above. If you want to type just ``tahoe`` instead
of ``venv\Scripts\tahoe``, you can either "`activate`_" the virtualenv (by
running ``venv\Scripts\activate``, or you can add the Scripts directory to
your ``%PATH%`` environment variable.
Now use the docs in :doc:`running` to learn how to configure your first
Tahoe node.
.. _activate: https://virtualenv.pypa.io/en/latest/userguide.html#activate-script
Installing A Different Version
------------------------------
The ``pip install tahoe-lafs`` command above will install the latest release
(from PyPI). If instead, you want to install from a git checkout, then run
the following command (using pip from the virtualenv, from the root of your
git checkout)::
$ venv\Scripts\pip install .
If you're planning to hack on the source code, you might want to add
``--editable`` so you won't have to re-install each time you make a change.
Dependencies
------------
Tahoe-LAFS depends upon several packages that use compiled C code (such as zfec).
This code must be built separately for each platform (Windows, OS-X, and different flavors of Linux).
Fortunately, this is now done by upstream packages for most platforms.
The result is that a C compiler is usually not required to install Tahoe-LAFS.

View File

@ -1,5 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
import shutil
from time import sleep
@ -28,7 +38,7 @@ from twisted.internet.error import (
import pytest
import pytest_twisted
from util import (
from .util import (
_CollectOutputProtocol,
_MagicTextProtocol,
_DumpOutputProtocol,

View File

@ -5,6 +5,15 @@
# You can safely skip any of these tests, it'll just appear to "take
# longer" to start the first test as the fixtures get built
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
def test_create_flogger(flog_gatherer):
print("Created flog_gatherer")

View File

@ -0,0 +1,64 @@
"""
Integration tests for getting and putting files, including reading from stdin
and stdout.
"""
from subprocess import Popen, PIPE
import pytest
from .util import run_in_thread, cli
DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11"
try:
DATA.decode("utf-8")
except UnicodeDecodeError:
pass # great, what we want
else:
raise ValueError("BUG, the DATA string was decoded from UTF-8")
@pytest.fixture(scope="session")
def get_put_alias(alice):
cli(alice, "create-alias", "getput")
def read_bytes(path):
with open(path, "rb") as f:
return f.read()
@run_in_thread
def test_put_from_stdin(alice, get_put_alias, tmpdir):
"""
It's possible to upload a file via `tahoe put`'s STDIN, and then download
it to a file.
"""
tempfile = str(tmpdir.join("file"))
p = Popen(
["tahoe", "--node-directory", alice.node_dir, "put", "-", "getput:fromstdin"],
stdin=PIPE
)
p.stdin.write(DATA)
p.stdin.close()
assert p.wait() == 0
cli(alice, "get", "getput:fromstdin", tempfile)
assert read_bytes(tempfile) == DATA
def test_get_to_stdout(alice, get_put_alias, tmpdir):
"""
It's possible to upload a file, and then download it to stdout.
"""
tempfile = tmpdir.join("file")
with tempfile.open("wb") as f:
f.write(DATA)
cli(alice, "put", str(tempfile), "getput:tostdout")
p = Popen(
["tahoe", "--node-directory", alice.node_dir, "get", "getput:tostdout", "-"],
stdout=PIPE
)
assert p.stdout.read() == DATA
assert p.wait() == 0

244
integration/test_i2p.py Normal file
View File

@ -0,0 +1,244 @@
"""
Integration tests for I2P support.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
from os.path import join, exists
from os import mkdir
from time import sleep
if PY2:
def which(path):
# This will result in skipping I2P tests on Python 2. Oh well.
return None
else:
from shutil import which
from eliot import log_call
import pytest
import pytest_twisted
from . import util
from twisted.python.filepath import (
FilePath,
)
from twisted.internet.error import ProcessExitedAlready
from allmydata.test.common import (
write_introducer,
)
if which("docker") is None:
pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)
# Docker on Windows machines sometimes expects Windows-y Docker images, so just
# don't bother.
if sys.platform.startswith('win'):
pytest.skip('Skipping I2P tests on Windows', allow_module_level=True)
@pytest.fixture
def i2p_network(reactor, temp_dir, request):
"""Fixture to start up local i2pd."""
proto = util._MagicTextProtocol("ephemeral keys")
reactor.spawnProcess(
proto,
which("docker"),
(
"docker", "run", "-p", "7656:7656", "purplei2p/i2pd",
# Bad URL for reseeds, so it can't talk to other routers.
"--reseed.urls", "http://localhost:1/",
),
)
def cleanup():
try:
proto.transport.signalProcess("KILL")
util.block_with_timeout(proto.exited, reactor)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
util.block_with_timeout(proto.magic_seen, reactor, timeout=30)
@pytest.fixture
@log_call(
action_type=u"integration:i2p:introducer",
include_args=["temp_dir", "flog_gatherer"],
include_result=False,
)
def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
config = '''
[node]
nickname = introducer_i2p
web.port = 4561
log_gatherer.furl = {log_furl}
'''.format(log_furl=flog_gatherer)
intro_dir = join(temp_dir, 'introducer_i2p')
print("making introducer", intro_dir)
if not exists(intro_dir):
mkdir(intro_dir)
done_proto = util._ProcessExitedProtocol()
util._tahoe_runner_optional_coverage(
done_proto,
reactor,
request,
(
'create-introducer',
'--listen=i2p',
intro_dir,
),
)
pytest_twisted.blockon(done_proto.done)
# over-write the config file with our stuff
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
protocol = util._MagicTextProtocol('introducer running')
transport = util._tahoe_runner_optional_coverage(
protocol,
reactor,
request,
(
'run',
intro_dir,
),
)
def cleanup():
try:
transport.signalProcess('TERM')
util.block_with_timeout(protocol.exited, reactor)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
pytest_twisted.blockon(protocol.magic_seen)
return transport
@pytest.fixture
def i2p_introducer_furl(i2p_introducer, temp_dir):
furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl')
while not exists(furl_fname):
print("Don't see {} yet".format(furl_fname))
sleep(.1)
furl = open(furl_fname, 'r').read()
return furl
@pytest_twisted.inlineCallbacks
def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
# ensure both nodes are connected to "a grid" by uploading
# something via carol, and retrieve it using dave.
gold_path = join(temp_dir, "gold")
with open(gold_path, "w") as f:
f.write(
"The object-capability model is a computer security model. A "
"capability describes a transferable right to perform one (or "
"more) operations on a given object."
)
# XXX could use treq or similar to POST these to their respective
# WUIs instead ...
proto = util._CollectOutputProtocol()
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'carol_i2p'),
'put', gold_path,
)
)
yield proto.done
cap = proto.output.getvalue().strip().split()[-1]
print("TEH CAP!", cap)
proto = util._CollectOutputProtocol(capture_stderr=False)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'dave_i2p'),
'get', cap,
)
)
yield proto.done
dave_got = proto.output.getvalue().strip()
assert dave_got == open(gold_path, 'rb').read().strip()
@pytest_twisted.inlineCallbacks
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl):
node_dir = FilePath(temp_dir).child(name)
web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
print("creating", node_dir.path)
node_dir.makedirs()
proto = util._DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'create-node',
'--nickname', name,
'--introducer', introducer_furl,
'--hide-ip',
'--listen', 'i2p',
node_dir.path,
)
)
yield proto.done
# Which services should this client connect to?
write_introducer(node_dir, "default", introducer_furl)
with node_dir.child('tahoe.cfg').open('w') as f:
node_config = '''
[node]
nickname = %(name)s
web.port = %(web_port)s
web.static = public_html
log_gatherer.furl = %(log_furl)s
[i2p]
enabled = true
[client]
shares.needed = 1
shares.happy = 1
shares.total = 2
''' % {
'name': name,
'web_port': web_port,
'log_furl': flog_gatherer,
}
node_config = node_config.encode("utf-8")
f.write(node_config)
print("running")
yield util._run_node(reactor, node_dir.path, request, None)
print("okay, launched")

View File

@ -1,9 +1,21 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
from os.path import join
from twisted.internet.error import ProcessTerminated
import util
from . import util
import pytest_twisted
@ -30,7 +42,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
proto,
sys.executable,
[
sys.executable, '-m', 'allmydata.scripts.runner',
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', node_dir,
'put', __file__,
]
@ -42,4 +54,4 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
assert isinstance(e, ProcessTerminated)
output = proto.output.getvalue()
assert "shares could be placed on only" in output
assert b"shares could be placed on only" in output

View File

@ -1,3 +1,6 @@
"""
Ported to Python 3.
"""
from __future__ import (
print_function,
unicode_literals,
@ -5,12 +8,18 @@ from __future__ import (
division,
)
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_text
import json
from os.path import (
join,
)
from urlparse import (
from urllib.parse import (
urlsplit,
)
@ -68,7 +77,7 @@ def _connect_client(reactor, api_auth_token, ws_url):
factory = WebSocketClientFactory(
url=ws_url,
headers={
"Authorization": "{} {}".format(SCHEME, api_auth_token),
"Authorization": "{} {}".format(str(SCHEME, "ascii"), api_auth_token),
}
)
factory.protocol = _StreamingLogClientProtocol
@ -127,7 +136,7 @@ def _test_streaming_logs(reactor, temp_dir, alice):
node_url = cfg.get_config_from_file("node.url")
api_auth_token = cfg.get_private_config("api_auth_token")
ws_url = node_url.replace("http://", "ws://")
ws_url = ensure_text(node_url).replace("http://", "ws://")
log_url = ws_url + "private/logs/v1"
print("Connecting to {}".format(log_url))

View File

@ -1,12 +1,22 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
from os.path import join
import pytest
import pytest_twisted
import util
from . import util
from twisted.python.filepath import (
FilePath,
@ -46,7 +56,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
proto,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'carol'),
'put', gold_path,
)
@ -55,12 +65,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
cap = proto.output.getvalue().strip().split()[-1]
print("TEH CAP!", cap)
proto = util._CollectOutputProtocol()
proto = util._CollectOutputProtocol(capture_stderr=False)
reactor.spawnProcess(
proto,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'-d', join(temp_dir, 'dave'),
'get', cap,
)
@ -68,7 +78,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
yield proto.done
dave_got = proto.output.getvalue().strip()
assert dave_got == open(gold_path, 'r').read().strip()
assert dave_got == open(gold_path, 'rb').read().strip()
@pytest_twisted.inlineCallbacks
@ -84,7 +94,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
proto,
sys.executable,
(
sys.executable, '-m', 'allmydata.scripts.runner',
sys.executable, '-b', '-m', 'allmydata.scripts.runner',
'create-node',
'--nickname', name,
'--introducer', introducer_furl,
@ -100,7 +110,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
# Which services should this client connect to?
write_introducer(node_dir, "default", introducer_furl)
with node_dir.child('tahoe.cfg').open('w') as f:
f.write('''
node_config = '''
[node]
nickname = %(name)s
web.port = %(web_port)s
@ -125,7 +135,9 @@ shares.total = 2
'log_furl': flog_gatherer,
'control_port': control_port,
'local_port': control_port + 1000,
})
}
node_config = node_config.encode("utf-8")
f.write(node_config)
print("running")
yield util._run_node(reactor, node_dir.path, request, None)

View File

@ -7,15 +7,26 @@ Most of the tests have cursory asserts and encode 'what the WebAPI did
at the time of testing' -- not necessarily a cohesive idea of what the
WebAPI *should* do in every situation. It's not clear the latter
exists anywhere, however.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
import json
import urllib2
from urllib.parse import unquote as url_unquote, quote as url_quote
import allmydata.uri
from allmydata.util import jsonbytes as json
import util
from . import util
import requests
import html5lib
@ -64,7 +75,7 @@ def test_upload_download(alice):
u"filename": u"boom",
}
)
assert data == FILE_CONTENTS
assert str(data, "utf-8") == FILE_CONTENTS
def test_put(alice):
@ -95,7 +106,7 @@ def test_helper_status(storage_nodes):
resp = requests.get(url)
assert resp.status_code >= 200 and resp.status_code < 300
dom = BeautifulSoup(resp.content, "html5lib")
assert unicode(dom.h1.string) == u"Helper Status"
assert str(dom.h1.string) == u"Helper Status"
def test_deep_stats(alice):
@ -115,10 +126,10 @@ def test_deep_stats(alice):
# when creating a directory, we'll be re-directed to a URL
# containing our writecap..
uri = urllib2.unquote(resp.url)
uri = url_unquote(resp.url)
assert 'URI:DIR2:' in uri
dircap = uri[uri.find("URI:DIR2:"):].rstrip('/')
dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(urllib2.quote(dircap)))
dircap_uri = util.node_url(alice.node_dir, "uri/{}".format(url_quote(dircap)))
# POST a file into this directory
FILE_CONTENTS = u"a file in a directory"
@ -145,7 +156,7 @@ def test_deep_stats(alice):
k, data = d
assert k == u"dirnode"
assert len(data['children']) == 1
k, child = data['children'].values()[0]
k, child = list(data['children'].values())[0]
assert k == u"filenode"
assert child['size'] == len(FILE_CONTENTS)
@ -196,11 +207,11 @@ def test_status(alice):
print("Uploaded data, cap={}".format(cap))
resp = requests.get(
util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap))),
util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap))),
)
print("Downloaded {} bytes of data".format(len(resp.content)))
assert resp.content == FILE_CONTENTS
assert str(resp.content, "ascii") == FILE_CONTENTS
resp = requests.get(
util.node_url(alice.node_dir, "status"),
@ -219,12 +230,12 @@ def test_status(alice):
continue
resp = requests.get(util.node_url(alice.node_dir, href))
if href.startswith(u"/status/up"):
assert "File Upload Status" in resp.content
if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
assert b"File Upload Status" in resp.content
if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
found_upload = True
elif href.startswith(u"/status/down"):
assert "File Download Status" in resp.content
if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
assert b"File Download Status" in resp.content
if b"Total Size: %d" % (len(FILE_CONTENTS),) in resp.content:
found_download = True
# download the specialized event information
@ -297,7 +308,7 @@ def test_directory_deep_check(alice):
print("Uploaded data1, cap={}".format(cap1))
resp = requests.get(
util.node_url(alice.node_dir, u"uri/{}".format(urllib2.quote(cap0))),
util.node_url(alice.node_dir, u"uri/{}".format(url_quote(cap0))),
params={u"t": u"info"},
)
@ -398,9 +409,9 @@ def test_directory_deep_check(alice):
for _ in range(5):
resp = requests.get(deepcheck_uri)
dom = BeautifulSoup(resp.content, "html5lib")
if dom.h1 and u'Results' in unicode(dom.h1.string):
if dom.h1 and u'Results' in str(dom.h1.string):
break
if dom.h2 and dom.h2.a and u"Reload" in unicode(dom.h2.a.string):
if dom.h2 and dom.h2.a and u"Reload" in str(dom.h2.a.string):
dom = None
time.sleep(1)
assert dom is not None, "Operation never completed"
@ -438,7 +449,7 @@ def test_introducer_info(introducer):
resp = requests.get(
util.node_url(introducer.node_dir, u""),
)
assert "Introducer" in resp.content
assert b"Introducer" in resp.content
resp = requests.get(
util.node_url(introducer.node_dir, u""),
@ -511,6 +522,6 @@ def test_mkdir_with_children(alice):
params={u"t": "mkdir-with-children"},
data=json.dumps(meta),
)
assert resp.startswith("URI:DIR2")
assert resp.startswith(b"URI:DIR2")
cap = allmydata.uri.from_string(resp)
assert isinstance(cap, allmydata.uri.DirectoryURI)

View File

@ -1,9 +1,21 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import sys
import time
import json
from os import mkdir, environ
from os.path import exists, join
from six.moves import StringIO
from io import StringIO, BytesIO
from functools import partial
from subprocess import check_output
@ -55,9 +67,10 @@ class _CollectOutputProtocol(ProcessProtocol):
self.output, and callback's on done with all of it after the
process exits (for any reason).
"""
def __init__(self):
def __init__(self, capture_stderr=True):
self.done = Deferred()
self.output = StringIO()
self.output = BytesIO()
self.capture_stderr = capture_stderr
def processEnded(self, reason):
if not self.done.called:
@ -71,7 +84,8 @@ class _CollectOutputProtocol(ProcessProtocol):
self.output.write(data)
def errReceived(self, data):
print("ERR: {}".format(data))
print("ERR: {!r}".format(data))
if self.capture_stderr:
self.output.write(data)
@ -92,9 +106,11 @@ class _DumpOutputProtocol(ProcessProtocol):
self.done.errback(reason)
def outReceived(self, data):
data = str(data, sys.stdout.encoding)
self._out.write(data)
def errReceived(self, data):
data = str(data, sys.stdout.encoding)
self._out.write(data)
@ -114,6 +130,7 @@ class _MagicTextProtocol(ProcessProtocol):
self.exited.callback(None)
def outReceived(self, data):
data = str(data, sys.stdout.encoding)
sys.stdout.write(data)
self._output.write(data)
if not self.magic_seen.called and self._magic_text in self._output.getvalue():
@ -121,6 +138,7 @@ class _MagicTextProtocol(ProcessProtocol):
self.magic_seen.callback(self)
def errReceived(self, data):
data = str(data, sys.stderr.encoding)
sys.stdout.write(data)
@ -152,9 +170,9 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
`--coverage` option if the `request` indicates we should.
"""
if request.config.getoption('coverage'):
args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
else:
args = [sys.executable, '-m', 'allmydata.scripts.runner']
args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner']
args += other_args
return reactor.spawnProcess(
proto,
@ -261,9 +279,9 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
'--hostname', 'localhost',
'--listen', 'tcp',
'--webport', web_port,
'--shares-needed', unicode(needed),
'--shares-happy', unicode(happy),
'--shares-total', unicode(total),
'--shares-needed', str(needed),
'--shares-happy', str(happy),
'--shares-total', str(total),
'--helper',
]
if not storage:
@ -280,7 +298,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
config,
u'node',
u'log_gatherer.furl',
flog_gatherer.decode("utf-8"),
flog_gatherer,
)
write_config(FilePath(config_path), config)
created_d.addCallback(created)
@ -526,7 +544,8 @@ def generate_ssh_key(path):
key = RSAKey.generate(2048)
key.write_private_key_file(path)
with open(path + ".pub", "wb") as f:
f.write(b"%s %s" % (key.get_name(), key.get_base64()))
s = "%s %s" % (key.get_name(), key.get_base64())
f.write(s.encode("ascii"))
def run_in_thread(f):

View File

@ -1,13 +1,18 @@
#! /usr/bin/python
# ./check-debugging.py src
"""
Checks for defer.setDebugging().
Runs on Python 3.
Usage: ./check-debugging.py src
"""
from __future__ import print_function
import sys, re, os
ok = True
umids = {}
for starting_point in sys.argv[1:]:
for root, dirs, files in os.walk(starting_point):

View File

@ -1,186 +0,0 @@
#! /usr/bin/python
from __future__ import print_function
import os, sys, compiler
from compiler.ast import Node, For, While, ListComp, AssName, Name, Lambda, Function
def check_source(source):
return check_thing(compiler.parse, source)
def check_file(path):
return check_thing(compiler.parseFile, path)
def check_thing(parser, thing):
try:
ast = parser(thing)
except SyntaxError as e:
return e
else:
results = []
check_ast(ast, results)
return results
def check_ast(ast, results):
"""Check a node outside a loop."""
if isinstance(ast, (For, While, ListComp)):
check_loop(ast, results)
else:
for child in ast.getChildNodes():
if isinstance(ast, Node):
check_ast(child, results)
def check_loop(ast, results):
"""Check a particular outer loop."""
# List comprehensions have a poorly designed AST of the form
# ListComp(exprNode, [ListCompFor(...), ...]), in which the
# result expression is outside the ListCompFor node even though
# it is logically inside the loop(s).
# There may be multiple ListCompFor nodes (in cases such as
# [lambda: (a,b) for a in ... for b in ...]
# ), and that case they are not nested in the AST. But these
# warts (nonobviously) happen not to matter for our analysis.
assigned = {} # maps name to lineno of topmost assignment
nested = set()
collect_assigned_and_nested(ast, assigned, nested)
# For each nested function...
for funcnode in nested:
# Check for captured variables in this function.
captured = set()
collect_captured(funcnode, assigned, captured, False)
for name in captured:
# We want to report the outermost capturing function
# (since that is where the workaround will need to be
# added), and the topmost assignment to the variable.
# Just one report per capturing function per variable
# will do.
results.append(make_result(funcnode, name, assigned[name]))
# Check each node in the function body in case it
# contains another 'for' loop.
childnodes = funcnode.getChildNodes()[len(funcnode.defaults):]
for child in childnodes:
check_ast(child, results)
def collect_assigned_and_nested(ast, assigned, nested):
"""
Collect the names assigned in this loop, not including names
assigned in nested functions. Also collect the nodes of functions
that are nested one level deep.
"""
if isinstance(ast, AssName):
if ast.name not in assigned or assigned[ast.name] > ast.lineno:
assigned[ast.name] = ast.lineno
else:
childnodes = ast.getChildNodes()
if isinstance(ast, (Lambda, Function)):
nested.add(ast)
# The default argument expressions are "outside" the
# function, even though they are children of the
# Lambda or Function node.
childnodes = childnodes[:len(ast.defaults)]
for child in childnodes:
if isinstance(ast, Node):
collect_assigned_and_nested(child, assigned, nested)
def collect_captured(ast, assigned, captured, in_function_yet):
"""Collect any captured variables that are also in assigned."""
if isinstance(ast, Name):
if ast.name in assigned:
captured.add(ast.name)
else:
childnodes = ast.getChildNodes()
if isinstance(ast, (Lambda, Function)):
# Formal parameters of the function are excluded from
# captures we care about in subnodes of the function body.
new_assigned = assigned.copy()
remove_argnames(ast.argnames, new_assigned)
if len(new_assigned) > 0:
for child in childnodes[len(ast.defaults):]:
collect_captured(child, new_assigned, captured, True)
# The default argument expressions are "outside" *this*
# function, even though they are children of the Lambda or
# Function node.
if not in_function_yet:
return
childnodes = childnodes[:len(ast.defaults)]
for child in childnodes:
if isinstance(ast, Node):
collect_captured(child, assigned, captured, True)
def remove_argnames(names, fromset):
for element in names:
if element in fromset:
del fromset[element]
elif isinstance(element, (tuple, list)):
remove_argnames(element, fromset)
def make_result(funcnode, var_name, var_lineno):
if hasattr(funcnode, 'name'):
func_name = 'function %r' % (funcnode.name,)
else:
func_name = '<lambda>'
return (funcnode.lineno, func_name, var_name, var_lineno)
def report(out, path, results):
for r in results:
print(path + (":%r %s captures %r assigned at line %d" % r), file=out)
def check(sources, out):
class Counts(object):
n = 0
processed_files = 0
suspect_files = 0
error_files = 0
counts = Counts()
def _process(path):
results = check_file(path)
if isinstance(results, SyntaxError):
print(path + (" NOT ANALYSED due to syntax error: %s" % results), file=out)
counts.error_files += 1
else:
report(out, path, results)
counts.n += len(results)
counts.processed_files += 1
if len(results) > 0:
counts.suspect_files += 1
for source in sources:
print("Checking %s..." % (source,), file=out)
if os.path.isfile(source):
_process(source)
else:
for (dirpath, dirnames, filenames) in os.walk(source):
for fn in filenames:
(basename, ext) = os.path.splitext(fn)
if ext == '.py':
_process(os.path.join(dirpath, fn))
print("%d suspiciously captured variables in %d out of %d file(s)."
% (counts.n, counts.suspect_files, counts.processed_files), file=out)
if counts.error_files > 0:
print("%d file(s) not processed due to syntax errors."
% (counts.error_files,), file=out)
return counts.n
sources = ['src']
if len(sys.argv) > 1:
sources = sys.argv[1:]
if check(sources, sys.stderr) > 0:
sys.exit(1)
# TODO: self-tests

View File

@ -1,4 +1,10 @@
#! /usr/bin/python
#! /usr/bin/python3
"""
Ensure UMIDS are unique.
This runs on Python 3.
"""
# ./check-umids.py src

View File

@ -52,6 +52,8 @@ system where Tahoe is installed, or in a source tree with setup.py like this:
setup.py run_with_pythonpath -p -c 'misc/make-canary-files.py ARGS..'
"""
from past.builtins import cmp
import os, hashlib
from twisted.python import usage
from allmydata.immutable import upload

View File

@ -201,7 +201,9 @@ class CPUWatcher(service.MultiService, resource.Resource, Referenceable):
log.msg("error reading process %s (%s), ignoring" % (pid, name))
log.err()
try:
pickle.dump(self.history, open("history.pickle.tmp", "wb"))
# Newer protocols won't work in Python 2; when it is dropped,
# protocol v4 can be used (added in Python 3.4).
pickle.dump(self.history, open("history.pickle.tmp", "wb"), protocol=2)
os.rename("history.pickle.tmp", "history.pickle")
except:
pass

View File

@ -18,7 +18,7 @@ def factorial(n):
factorial(n) with n<0 is -factorial(abs(n))
"""
result = 1
for i in xrange(1, abs(n)+1):
for i in range(1, abs(n)+1):
result *= i
assert n >= 0
return result
@ -30,7 +30,7 @@ def binomial(n, k):
# calculate n!/k! as one product, avoiding factors that
# just get canceled
P = k+1
for i in xrange(k+2, n+1):
for i in range(k+2, n+1):
P *= i
# if you are paranoid:
# C, rem = divmod(P, factorial(n-k))

View File

@ -30,7 +30,7 @@ class Root(rend.Page):
def run(portnum):
root = Root()
root.putChild("tahoe.css", static.File("tahoe.css"))
root.putChild(b"tahoe.css", static.File("tahoe.css"))
site = appserver.NevowSite(root)
s = strports.service("tcp:%d" % portnum, site)
s.startService()

View File

@ -79,7 +79,7 @@ def make_candidate(B, K, K1, K2, q, T, T_min, L_hash, lg_N, sig_bytes, c_sign, c
# Winternitz with B < 4 is never optimal. For example, going from B=4 to B=2 halves the
# chain depth, but that is cancelled out by doubling (roughly) the number of digits.
range_B = xrange(4, 33)
range_B = range(4, 33)
M = pow(2, lg_M)
@ -100,7 +100,7 @@ def calculate(K, K1, K2, q_max, L_hash, trees):
T_min = ceil_div(lg_M - lg_K1, lg_K)
last_q = None
for T in xrange(T_min, T_min+21):
for T in range(T_min, T_min+21):
# lg(total number of leaf private keys)
lg_S = lg_K1 + lg_K*T
lg_N = lg_S + lg_K2
@ -137,14 +137,14 @@ def calculate(K, K1, K2, q_max, L_hash, trees):
# We approximate lg(M-x) as lg(M)
lg_px_step = lg_M + lg_p - lg_1_p
for x in xrange(1, j):
for x in range(1, j):
lg_px[x] = lg_px[x-1] - lg(x) + lg_px_step
q = None
# Find the minimum acceptable value of q.
for q_cand in xrange(1, q_max+1):
for q_cand in range(1, q_max+1):
lg_q = lg(q_cand)
lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in xrange(1, j)]
lg_pforge = [lg_px[x] + (lg_q*x - lg_K2)*q_cand for x in range(1, j)]
if max(lg_pforge) < -L_hash + lg(j) and lg_px[j-1] + 1.0 < -L_hash:
#print("K = %d, K1 = %d, K2 = %d, L_hash = %d, lg_K2 = %.3f, q = %d, lg_pforge_1 = %.3f, lg_pforge_2 = %.3f, lg_pforge_3 = %.3f"
# % (K, K1, K2, L_hash, lg_K2, q, lg_pforge_1, lg_pforge_2, lg_pforge_3))
@ -246,13 +246,13 @@ def search():
K_max = 50
c2 = compressions(2*L_hash)
c3 = compressions(3*L_hash)
for dau in xrange(0, 10):
for dau in range(0, 10):
a = pow(2, dau)
for tri in xrange(0, ceil_log(30-dau, 3)):
for tri in range(0, ceil_log(30-dau, 3)):
x = int(a*pow(3, tri))
h = dau + 2*tri
c_x = int(sum_powers(2, dau)*c2 + a*sum_powers(3, tri)*c3)
for y in xrange(1, x+1):
for y in range(1, x+1):
if tri > 0:
# If the bottom level has arity 3, then for every 2 nodes by which the tree is
# imperfect, we can save c3 compressions by pruning 3 leaves back to their parent.
@ -267,16 +267,16 @@ def search():
if y not in trees or (h, c_y, (dau, tri)) < trees[y]:
trees[y] = (h, c_y, (dau, tri))
#for x in xrange(1, K_max+1):
#for x in range(1, K_max+1):
# print(x, trees[x])
candidates = []
progress = 0
fuzz = 0
complete = (K_max-1)*(2200-200)/100
for K in xrange(2, K_max+1):
for K2 in xrange(200, 2200, 100):
for K1 in xrange(max(2, K-fuzz), min(K_max, K+fuzz)+1):
for K in range(2, K_max+1):
for K2 in range(200, 2200, 100):
for K1 in range(max(2, K-fuzz), min(K_max, K+fuzz)+1):
candidates += calculate(K, K1, K2, q_max, L_hash, trees)
progress += 1
print("searching: %3d %% \r" % (100.0 * progress / complete,), end=' ', file=stderr)
@ -285,7 +285,7 @@ def search():
step = 2.0
bins = {}
limit = floor_div(limit_cost, step)
for bin in xrange(0, limit+2):
for bin in range(0, limit+2):
bins[bin] = []
for c in candidates:
@ -296,7 +296,7 @@ def search():
# For each in a range of signing times, find the best candidate.
best = []
for bin in xrange(0, limit):
for bin in range(0, limit):
candidates = bins[bin] + bins[bin+1] + bins[bin+2]
if len(candidates) > 0:
best += [min(candidates, key=lambda c: c['sig_bytes'])]

View File

@ -4,6 +4,8 @@
from __future__ import print_function
from past.builtins import cmp
import random
SERVER_CAPACITY = 10**12

View File

@ -2,6 +2,11 @@
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import input
import random, math, re
from twisted.python import usage
@ -205,7 +210,7 @@ def graph():
series["alacrity"][file_size] = s.bytes_until_some_data
g.plot([ (fs, series["overhead"][fs])
for fs in sizes ])
raw_input("press return")
input("press return")
if __name__ == '__main__':

View File

@ -1 +0,0 @@
PyPy is now a supported platform.

View File

@ -1 +0,0 @@
The Tahoe-LAFS project has adopted a formal code of conduct.

1
newsfragments/3037.other Normal file
View File

@ -0,0 +1 @@
The "Great Black Swamp" proposed specification has been expanded to include two lease management APIs.

View File

@ -1 +0,0 @@
The Magic Folder frontend has been split out into a stand-alone project. The functionality is no longer part of Tahoe-LAFS itself. Learn more at <https://github.com/LeastAuthority/magic-folder>.

View File

@ -1 +0,0 @@
Tahoe-LAFS now supports CentOS 8 and no longer supports CentOS 7.

View File

@ -1 +0,0 @@
Make directory page links work.

View File

@ -1 +0,0 @@
Replace nevow with twisted.web in web.operations.OphandleTable

View File

@ -1 +0,0 @@
Replace nevow with twisted.web in web.operations.ReloadMixin

View File

@ -1 +0,0 @@
Port checker result pages' rendering from nevow to twisted web templates.

View File

@ -1 +0,0 @@
allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients.

View File

@ -1 +0,0 @@
Slackware 14.2 is no longer a Tahoe-LAFS supported platform.

View File

@ -1 +0,0 @@
Tahoe-LAFS now supports Ubuntu 20.04.

View File

@ -1 +0,0 @@
Use last known revision of Chutney that is known to work with Python 2 for Tor integration tests.

View File

@ -1 +0,0 @@
Mutable files now use RSA exponent 65537

View File

@ -1 +0,0 @@
The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

View File

@ -1 +0,0 @@

View File

@ -1 +0,0 @@

View File

@ -1 +0,0 @@
Added pre-commit config to run flake8 checks on commit/push.

View File

@ -1 +0,0 @@
Various, minor development `./Makefile` cleanup and improvement.

View File

@ -1 +0,0 @@
Minor test runner improvements and docs.

View File

@ -1 +0,0 @@
Convert modules that only reference `unicode` to use `str`.

View File

@ -1 +0,0 @@
Begin porting the `node` module to Python 3.

View File

@ -1 +0,0 @@
Cleanup comments that don't match the project convention.

Some files were not shown because too many files have changed in this diff Show More