ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-18 20:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into release-1.15.0-rc0

Browse Source
This commit is contained in:
meejah 2021-02-02 12:14:13 -07:00
commit 00613b78f5
335 changed files with 10104 additions and 8240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
.circleci/config.yml
View File

@ -14,44 +14,73 @@ version: 2.1
workflows:
ci:
jobs:
# Platforms
- "debian-9"
# Start with jobs testing various platforms.
# Every job that pulls a Docker image from Docker Hub needs to provide
# credentials for that pull operation to avoid being subjected to
# unauthenticated pull limits shared across all of CircleCI. Use this
# first job to define a yaml anchor that can be used to supply a
# CircleCI job context which makes Docker Hub credentials available in
# the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
- "debian-9": &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
- "debian-8":
<<: *DOCKERHUB_CONTEXT
requires:
- "debian-9"
- "ubuntu-20-04"
- "ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "ubuntu-16-04":
<<: *DOCKERHUB_CONTEXT
requires:
- "ubuntu-20-04"
- "fedora-29"
- "fedora-29":
<<: *DOCKERHUB_CONTEXT
- "fedora-28":
<<: *DOCKERHUB_CONTEXT
requires:
- "fedora-29"
- "centos-8"
- "centos-8":
<<: *DOCKERHUB_CONTEXT
- "nixos-19-09"
- "nixos-19-09":
<<: *DOCKERHUB_CONTEXT
# Test against PyPy 2.7
- "pypy27-buster"
- "pypy27-buster":
<<: *DOCKERHUB_CONTEXT
# Just one Python 3.6 configuration while the port is in-progress.
- "python36"
- "python36":
<<: *DOCKERHUB_CONTEXT
# Other assorted tasks and configurations
- "lint"
- "pyinstaller"
- "deprecations"
- "c-locale"
- "lint":
<<: *DOCKERHUB_CONTEXT
- "pyinstaller":
<<: *DOCKERHUB_CONTEXT
- "deprecations":
<<: *DOCKERHUB_CONTEXT
- "c-locale":
<<: *DOCKERHUB_CONTEXT
# Any locale other than C or UTF-8.
- "another-locale"
- "another-locale":
<<: *DOCKERHUB_CONTEXT
- "integration":
<<: *DOCKERHUB_CONTEXT
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
@ -59,7 +88,11 @@ workflows:
# Generate the underlying data for a visualization to aid with Python 3
# porting.
- "build-porting-depgraph"
- "build-porting-depgraph":
<<: *DOCKERHUB_CONTEXT
- "typechecks":
<<: *DOCKERHUB_CONTEXT
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
@ -74,22 +107,55 @@ workflows:
- "master"
jobs:
- "build-image-debian-8"
- "build-image-debian-9"
- "build-image-ubuntu-16-04"
- "build-image-ubuntu-18-04"
- "build-image-ubuntu-20-04"
- "build-image-fedora-28"
- "build-image-fedora-29"
- "build-image-centos-8"
- "build-image-pypy27-buster"
- "build-image-python36-ubuntu"
- "build-image-debian-8":
<<: *DOCKERHUB_CONTEXT
- "build-image-debian-9":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-16-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-18-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-28":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-29":
<<: *DOCKERHUB_CONTEXT
- "build-image-centos-8":
<<: *DOCKERHUB_CONTEXT
- "build-image-pypy27-buster":
<<: *DOCKERHUB_CONTEXT
- "build-image-python36-ubuntu":
<<: *DOCKERHUB_CONTEXT
jobs:
dockerhub-auth-template:
# This isn't a real job. It doesn't get scheduled as part of any
# workflow. Instead, it's just a place we can hang a yaml anchor to
# finish the Docker Hub authentication configuration. Workflow jobs using
# the DOCKERHUB_CONTEXT anchor will have access to the environment
# variables used here. These variables will allow the Docker Hub image
# pull to be authenticated and hopefully avoid hitting and rate limits.
docker: &DOCKERHUB_AUTH
- image: "null"
auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
steps:
- run:
name: "CircleCI YAML schema conformity"
command: |
# This isn't a real command. We have to have something in this
# space, though, or the CircleCI yaml schema validator gets angry.
# Since this job is never scheduled this step is never run so the
# actual value here is irrelevant.
lint:
docker:
- image: "circleci/python:2"
- <<: *DOCKERHUB_AUTH
image: "circleci/python:2"
steps:
- "checkout"
@ -106,7 +172,8 @@ jobs:
pyinstaller:
docker:
- image: "circleci/python:2"
- <<: *DOCKERHUB_AUTH
image: "circleci/python:2"
steps:
- "checkout"
@ -131,7 +198,8 @@ jobs:
debian-9: &DEBIAN
docker:
- image: "tahoelafsci/debian:9-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:9-py2.7"
user: "nobody"
environment: &UTF_8_ENVIRONMENT
@ -154,6 +222,8 @@ jobs:
# we maintain.
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
# Upload the coverage report.
UPLOAD_COVERAGE: "yes"
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
@ -202,26 +272,32 @@ jobs:
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
command: |
/tmp/venv/bin/codecov
if [ -n "${UPLOAD_COVERAGE}" ]; then
/tmp/venv/bin/codecov
fi
debian-8:
<<: *DEBIAN
docker:
- image: "tahoelafsci/debian:8-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:8-py2.7"
user: "nobody"
pypy27-buster:
<<: *DEBIAN
docker:
- image: "tahoelafsci/pypy:buster-py2"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/pypy:buster-py2"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# We don't do coverage since it makes PyPy far too slow:
TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
# Since we didn't collect it, don't upload it.
UPLOAD_COVERAGE: ""
c-locale:
@ -250,6 +326,8 @@ jobs:
TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
# Put the logs somewhere we can report them.
TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
# The deprecations tox environments don't do coverage measurement.
UPLOAD_COVERAGE: ""
integration:
@ -272,21 +350,24 @@ jobs:
ubuntu-16-04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:16.04-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:16.04-py2.7"
user: "nobody"
ubuntu-18-04: &UBUNTU_18_04
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:18.04-py2.7"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py2.7"
user: "nobody"
python36:
<<: *UBUNTU_18_04
docker:
- image: "tahoelafsci/ubuntu:18.04-py3"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3"
user: "nobody"
environment:
@ -301,13 +382,15 @@ jobs:
ubuntu-20-04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:20.04"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:20.04"
user: "nobody"
centos-8: &RHEL_DERIV
docker:
- image: "tahoelafsci/centos:8-py2"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/centos:8-py2"
user: "nobody"
environment: *UTF_8_ENVIRONMENT
@ -329,21 +412,24 @@ jobs:
fedora-28:
<<: *RHEL_DERIV
docker:
- image: "tahoelafsci/fedora:28-py"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/fedora:28-py"
user: "nobody"
fedora-29:
<<: *RHEL_DERIV
docker:
- image: "tahoelafsci/fedora:29-py"
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/fedora:29-py"
user: "nobody"
nixos-19-09:
docker:
# Run in a highly Nix-capable environment.
- image: "nixorg/nix:circleci"
- <<: *DOCKERHUB_AUTH
image: "nixorg/nix:circleci"
environment:
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
@ -392,6 +478,18 @@ jobs:
. /tmp/venv/bin/activate
./misc/python3/depgraph.sh
typechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:18.04-py3"
steps:
- "checkout"
- run:
name: "Validate Types"
command: |
/tmp/venv/bin/tox -e typechecks
build-image: &BUILD_IMAGE
# This is a template for a job to build a Docker image that has as much of
# the setup as we can manage already done and baked in. This cuts down on
@ -400,7 +498,8 @@ jobs:
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- image: "docker:17.05.0-ce-git"
- <<: *DOCKERHUB_AUTH
image: "docker:17.05.0-ce-git"
environment:
DISTRO: "tahoelafsci/<DISTRO>:foo-py2"
@ -410,47 +509,10 @@ jobs:
steps:
- "checkout"
- "setup_remote_docker"
- run:
name: "Get openssl"
command: |
apk add --no-cache openssl
- run:
name: "Get Dockerhub secrets"
command: |
# If you create an encryption key like this:
#
# openssl enc -aes-256-cbc -k secret -P -md sha256
# From the output that looks like:
#
# salt=...
# key=...
# iv =...
#
# extract just the value for ``key``.
# then you can re-generate ``secret-env-cipher`` locally using the
# command:
#
# openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
#
# Make sure the key is set as the KEY environment variable in the
# CircleCI web interface. You can do this by visiting
# <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
# after logging in to CircleCI with an account in the tahoe-lafs
# CircleCI team.
#
# Then you can recover the environment plaintext (for example, to
# change and re-encrypt it) like just like CircleCI recovers it
# here:
#
openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
- run:
name: "Log in to Dockerhub"
command: |
. ~/.env
# TAHOELAFSCI_PASSWORD come from the secret env.
docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
- run:
name: "Build image"
command: |

1
.circleci/secret-env-cipher
View File

@ -1 +0,0 @@
Salted__ •GPÁøÊ)|!÷[©U[‡ûvSÚ,F¿–m:ö š~ÓY[Uú_¸Fx×’¤Ÿ%<25>“4l×Ö»Š8¼œ¹„1öø‰/lƒÌ`nÆ^·Z]óqš¬æ¢&ø°÷£Ý‚‚ß%T¡n

14
.codecov.yml
View File

@ -32,3 +32,17 @@ coverage:
patch:
default:
threshold: 1%
codecov:
# This is a public repository so supposedly we don't "need" to use an upload
# token. However, using one makes sure that CI jobs running against forked
# repositories have coverage uploaded to the right place in codecov so
# their reports aren't incomplete.
token: "abf679b6-e2e6-4b33-b7b5-6cfbd41ee691"
notify:
# The reference documentation suggests that this is the default setting:
# https://docs.codecov.io/docs/codecovyml-reference#codecovnotifywait_for_ci
# However observation suggests otherwise.
wait_for_ci: true

11
.coveragerc
View File

@ -14,3 +14,14 @@ branch = True
[report]
show_missing = True
skip_covered = True
[paths]
source =
# It looks like this in the checkout
src/
# It looks like this in the Windows build environment
D:/a/tahoe-lafs/tahoe-lafs/.tox/py*-coverage/Lib/site-packages/
# Although sometimes it looks like this instead. Also it looks like this on macOS.
.tox/py*-coverage/lib/python*/site-packages/
# On some Linux CI jobs it looks like this
/tmp/tahoe-lafs.tox/py*-coverage/lib/python*/site-packages/

2
.github/CONTRIBUTING.rst vendored
View File

@ -17,4 +17,4 @@ Examples of contributions include:
* `Patch reviews <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/PatchReviewProcess>`_
Before authoring or reviewing a patch,
please familiarize yourself with the `coding standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`_.
please familiarize yourself with the `Coding Standards <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_.

81
.github/workflows/ci.yml vendored
View File

@ -21,7 +21,7 @@ jobs:
steps:
# Get vcpython27 on Windows + Python 2.7, to build zfec
# Get vcpython27 on Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
@ -30,17 +30,37 @@ jobs:
with:
args: install vcpython27
# See https://github.com/actions/checkout. A fetch-depth of 0
# fetches all tags and branches.
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
- name: Fetch all history for all tags and branches
run: git fetch --prune --unshallow
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
# To use pip caching with GitHub Actions in an OS-independent
# manner, we need `pip cache dir` command, which became
# available since pip v20.1+. At the time of writing this,
# GitHub Actions offers pip v20.3.3 for both ubuntu-latest and
# windows-latest, and pip v20.3.1 for macos-latest.
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
# See https://github.com/actions/cache
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install Python packages
run: |
pip install --upgrade codecov tox setuptools
@ -78,6 +98,15 @@ jobs:
steps:
# Get vcpython27 for Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
uses: crazy-max/ghaction-chocolatey@v1
with:
args: install vcpython27
- name: Install Tor [Ubuntu]
if: matrix.os == 'ubuntu-latest'
run: sudo apt install tor
@ -92,23 +121,29 @@ jobs:
with:
args: install tor
- name: Install MSVC 9.0 for Python 2.7 [Windows]
if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
uses: crazy-max/ghaction-chocolatey@v1
with:
args: install vcpython27
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
- name: Fetch all history for all tags and branches
run: git fetch --prune --unshallow
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install Python packages
run: |
pip install --upgrade tox
@ -141,7 +176,7 @@ jobs:
steps:
# Get vcpython27 on Windows + Python 2.7, to build zfec
# Get vcpython27 for Windows + Python 2.7, to build netifaces
# extension. See https://chocolatey.org/packages/vcpython27 and
# https://github.com/crazy-max/ghaction-chocolatey
- name: Install MSVC 9.0 for Python 2.7 [Windows]
@ -152,15 +187,27 @@ jobs:
- name: Check out Tahoe-LAFS sources
uses: actions/checkout@v2
- name: Fetch all history for all tags and branches
run: git fetch --prune --unshallow
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Get pip cache directory
id: pip-cache
run: |
echo "::set-output name=dir::$(pip cache dir)"
- name: Use pip cache
uses: actions/cache@v2
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install Python packages
run: |
pip install --upgrade tox

19
.pre-commit-config.yaml
View File

@ -1,15 +1,10 @@
repos:
- repo: local
- repo: "local"
hooks:
- id: codechecks
name: codechecks
stages: ["commit"]
entry: "tox -e codechecks"
language: system
pass_filenames: false
- id: test
name: test
- id: "codechecks"
name: "codechecks"
stages: ["push"]
entry: "make test"
language: system
pass_filenames: false
language: "system"
files: ".py$"
entry: "tox -e codechecks"
pass_filenames: true

11
CREDITS
View File

@ -113,6 +113,8 @@ E: jacob@appelbaum.com
W: http://www.appelbaum.net/
P: 12E4 04FF D3C9 31F9 3405 2D06 B884 1A91 9D0F ACE4
D: Debian packaging including init scripts
D: Note that contributions from Jacob Appelbaum (ioerror) are no longer welcome
D: due to behavior unacceptable to community standards in Tor and other projects
N: Jeremy Visser
D: Ubuntu packaging, usability testing
@ -229,3 +231,12 @@ D: Python3 porting
N: Pete Fein
E: pete@snake.dev
D: Python3 porting
N: Viktoriia Savchuk
W: https://twitter.com/viktoriiasvchk
D: Developer community focused improvements on the README file.
N: Lukas Pirl
E: tahoe@lukas-pirl.de
W: http://lukas-pirl.de
D: Buildslaves (Debian, Fedora, CentOS; 2016-2021)

16
Makefile
View File

@ -13,8 +13,6 @@ MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules
# Local target variables
VCS_HOOK_SAMPLES=$(wildcard .git/hooks/*.sample)
VCS_HOOKS=$(VCS_HOOK_SAMPLES:%.sample=%)
PYTHON=python
export PYTHON
PYFLAKES=flake8
@ -31,15 +29,6 @@ TEST_SUITE=allmydata
default:
@echo "no default target"
.PHONY: install-vcs-hooks
## Install the VCS hooks to run linters on commit and all tests on push
install-vcs-hooks: .git/hooks/pre-commit .git/hooks/pre-push
.PHONY: uninstall-vcs-hooks
## Remove the VCS hooks
uninstall-vcs-hooks: .tox/create-venvs.log
"./$(dir $(<))py36/bin/pre-commit" uninstall || true
"./$(dir $(<))py36/bin/pre-commit" uninstall -t pre-push || true
.PHONY: test
## Run all tests and code reports
test: .tox/create-venvs.log
@ -215,7 +204,7 @@ clean:
rm -f *.pkg
.PHONY: distclean
distclean: clean uninstall-vcs-hooks
distclean: clean
rm -rf src/*.egg-info
rm -f src/allmydata/_version.py
rm -f src/allmydata/_appname.py
@ -261,6 +250,3 @@ src/allmydata/_version.py:
.tox/create-venvs.log: tox.ini setup.py
tox --notest -p all | tee -a "$(@)"
$(VCS_HOOKS): .tox/create-venvs.log .pre-commit-config.yaml
"./$(dir $(<))py36/bin/pre-commit" install --hook-type $(@:.git/hooks/%=%)

152
README.rst
View File

@ -1,97 +1,119 @@
==========
Tahoe-LAFS
==========
======================================
Free and Open decentralized data store
======================================
Tahoe-LAFS is a Free and Open decentralized cloud storage system. It
distributes your data across multiple servers. Even if some of the servers
fail or are taken over by an attacker, the entire file store continues to
function correctly, preserving your privacy and security.
|image0|
For full documentation, please see
http://tahoe-lafs.readthedocs.io/en/latest/ .
`Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov|
INSTALLING
==========
Table of contents
There are three ways to install Tahoe-LAFS.
- `About Tahoe-LAFS <#about-tahoe-lafs>`__
using OS packages
^^^^^^^^^^^^^^^^^
- `Installation <#installation>`__
Pre-packaged versions are available for several operating systems:
- `Issues <#issues>`__
* Debian and Ubuntu users can ``apt-get install tahoe-lafs``
* NixOS, NetBSD (pkgsrc), ArchLinux, Slackware, and Gentoo have packages
available, see `OSPackages`_ for details
* `Mac`_ and Windows installers are in development.
- `Documentation <#documentation>`__
via pip
^^^^^^^
- `Community <#community>`__
If you don't use an OS package, you'll need Python 2.7 and `pip`_. You may
also need a C compiler, and the development headers for python, libffi, and
OpenSSL. On a Debian-like system, use ``apt-get install build-essential
python-dev libffi-dev libssl-dev python-virtualenv``. On Windows, see
`<docs/windows.rst>`_.
- `Contributing <#contributing>`__
Then, to install the most recent release, just run:
- `FAQ <#faq>`__
* ``pip install tahoe-lafs``
- `License <#license>`__
from source
^^^^^^^^^^^
To install from source (either so you can hack on it, or just to run
pre-release code), you should create a virtualenv and install into that:
💡 About Tahoe-LAFS
-------------------
* ``git clone https://github.com/tahoe-lafs/tahoe-lafs.git``
* ``cd tahoe-lafs``
* ``virtualenv --python=python2.7 venv``
* ``venv/bin/pip install --upgrade setuptools``
* ``venv/bin/pip install --editable .``
* ``venv/bin/tahoe --version``
Tahoe-LAFS helps you to store files while granting confidentiality, integrity, and availability of your data.
To run the unit test suite:
How does it work? You run a client program on your computer, which talks to one or more storage servers on other computers. When you tell your client to store a file, it will encrypt that file, encode it into multiple pieces, then spread those pieces out among various servers. The pieces are all encrypted and protected against modifications. Later, when you ask your client to retrieve the file, it will find the necessary pieces, make sure they haven’t been corrupted, reassemble them, and decrypt the result.
* ``tox``
| |image2|
| *The image is taken from meejah's* \ `blog <https://blog.torproject.org/tor-heart-tahoe-lafs>`__ \ *post at Torproject.org.*
You can pass arguments to ``trial`` with an environment variable. For
example, you can run the test suite on multiple cores to speed it up:
|
* ``TAHOE_LAFS_TRIAL_ARGS="-j4" tox``
The client creates pieces (“shares”) that have a configurable amount of redundancy, so even if some servers fail, you can still get your data back. Corrupt shares are detected and ignored so that the system can tolerate server-side hard-drive errors. All files are encrypted (with a unique key) before uploading, so even a malicious server operator cannot read your data. The only thing you ask of the servers is that they can (usually) provide the shares when you ask for them: you aren’t relying upon them for confidentiality, integrity, or absolute availability.
For more detailed instructions, read `<docs/INSTALL.rst>`_ .
Tahoe-LAFS was first designed in 2007, following the "principle of least authority", a security best practice requiring system components to only have the privilege necessary to complete their intended function and not more.
Once ``tahoe --version`` works, see `<docs/running.rst>`_ to learn how to set
up your first Tahoe-LAFS node.
Please read more about Tahoe-LAFS architecture `here <docs/architecture.rst>`__.
LICENCE
=======
✅ Installation
---------------
Copyright 2006-2018 The Tahoe-LAFS Software Foundation
For more detailed instructions, read `docs/INSTALL.rst <docs/INSTALL.rst>`__ .
You may use this package under the GNU General Public License, version 2 or,
at your option, any later version. You may use this package under the
Transitive Grace Period Public Licence, version 1.0, or at your option, any
later version. (You may choose to use this package under the terms of either
licence, at your option.) See the file `COPYING.GPL`_ for the terms of the
GNU General Public License, version 2. See the file `COPYING.TGPPL`_ for
the terms of the Transitive Grace Period Public Licence, version 1.0.
- `Building Tahoe-LAFS on Windows <docs/windows.rst>`__
See `TGPPL.PDF`_ for why the TGPPL exists, graphically illustrated on three
slides.
- `OS-X Packaging <docs/OS-X.rst>`__
.. _OSPackages: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/OSPackages
.. _Mac: docs/OS-X.rst
.. _pip: https://pip.pypa.io/en/stable/installing/
.. _COPYING.GPL: https://github.com/tahoe-lafs/tahoe-lafs/blob/master/COPYING.GPL
.. _COPYING.TGPPL: https://github.com/tahoe-lafs/tahoe-lafs/blob/master/COPYING.TGPPL.rst
.. _TGPPL.PDF: https://tahoe-lafs.org/~zooko/tgppl.pdf
Once tahoe --version works, see `docs/running.rst <docs/running.rst>`__ to learn how to set up your first Tahoe-LAFS node.
----
🤖 Issues
---------
Tahoe-LAFS uses the Trac instance to track `issues <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/ViewTickets>`__. Please email jean-paul plus tahoe-lafs at leastauthority dot com for an account.
📑 Documentation
----------------
You can find the full Tahoe-LAFS documentation at our `documentation site <http://tahoe-lafs.readthedocs.io/en/latest/>`__.
💬 Community
------------
Get involved with the Tahoe-LAFS community:
- Chat with Tahoe-LAFS developers at #tahoe-lafs chat on irc.freenode.net or `Slack <https://join.slack.com/t/tahoe-lafs/shared_invite/zt-jqfj12r5-ZZ5z3RvHnubKVADpP~JINQ>`__.
- Join our `weekly conference calls <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/WeeklyMeeting>`__ with core developers and interested community members.
- Subscribe to `the tahoe-dev mailing list <https://www.tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev>`__, the community forum for discussion of Tahoe-LAFS design, implementation, and usage.
🤗 Contributing
---------------
As a community-driven open source project, Tahoe-LAFS welcomes contributions of any form:
- `Code patches <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Patches>`__
- `Documentation improvements <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Doc>`__
- `Bug reports <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/HowToReportABug>`__
- `Patch reviews <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/PatchReviewProcess>`__
Before authoring or reviewing a patch, please familiarize yourself with the `Coding Standard <https://tahoe-lafs.org/trac/tahoe-lafs/wiki/CodingStandards>`__ and the `Contributor Code of Conduct <docs/CODE_OF_CONDUCT.md>`__.
❓ FAQ
------
Need more information? Please check our `FAQ page <https://www.tahoe-lafs.org/trac/tahoe-lafs/wiki/FAQ>`__.
📄 License
----------
Copyright 2006-2020 The Tahoe-LAFS Software Foundation
You may use this package under the GNU General Public License, version 2 or, at your option, any later version. You may use this package under the Transitive Grace Period Public Licence, version 1.0, or at your choice, any later version. (You may choose to use this package under the terms of either license, at your option.) See the file `COPYING.GPL <COPYING.GPL>`__ for the terms of the GNU General Public License, version 2. See the file `COPYING.TGPPL <COPYING.TGPPL.rst>`__ for the terms of the Transitive Grace Period Public Licence, version 1.0.
See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL exists, graphically illustrated on three slides.
.. |image0| image:: docs/_static/media/image2.png
:width: 3in
:height: 0.91667in
.. |image2| image:: docs/_static/media/image1.png
:width: 6.9252in
:height: 2.73611in
.. |readthedocs| image:: http://readthedocs.org/projects/tahoe-lafs/badge/?version=latest
:alt: documentation status
:target: http://tahoe-lafs.readthedocs.io/en/latest/?badge=latest

46
docs/INSTALL.rst
View File

@ -39,9 +39,7 @@ If you are on Windows, please see :doc:`windows` for platform-specific
instructions.
If you are on a Mac, you can either follow these instructions, or use the
pre-packaged bundle described in :doc:`OS-X`. The Tahoe project hosts
pre-compiled "wheels" for all dependencies, so use the ``--find-links=``
option described below to avoid needing a compiler.
pre-packaged bundle described in :doc:`OS-X`.
Many Linux distributions include Tahoe-LAFS packages. Debian and Ubuntu users
can ``apt-get install tahoe-lafs``. See `OSPackages`_ for other
@ -54,9 +52,14 @@ Preliminaries
=============
If you don't use a pre-packaged copy of Tahoe, you can build it yourself.
You'll need Python2.7, pip, and virtualenv. On unix-like platforms, you will
need a C compiler, the Python development headers, and some libraries
(libffi-dev and libssl-dev).
You'll need Python2.7, pip, and virtualenv.
Tahoe-LAFS depends on some libraries which require a C compiler to build.
However, for many platforms, PyPI hosts already-built packages of libraries.
If there is no already-built package for your platform,
you will need a C compiler,
the Python development headers,
and some libraries (libffi-dev and libssl-dev).
On a modern Debian/Ubuntu-derived distribution, this command will get you
everything you need::
@ -64,8 +67,7 @@ everything you need::
apt-get install build-essential python-dev libffi-dev libssl-dev libyaml-dev python-virtualenv
On OS-X, install pip and virtualenv as described below. If you want to
compile the dependencies yourself (instead of using ``--find-links`` to take
advantage of the pre-compiled ones we host), you'll also need to install
compile the dependencies yourself, you'll also need to install
Xcode and its command-line tools.
**Note** that Tahoe-LAFS depends on `openssl 1.1.1c` or greater.
@ -150,30 +152,24 @@ from PyPI with ``venv/bin/pip install tahoe-lafs``. After installation, run
% virtualenv venv
New python executable in ~/venv/bin/python2.7
Installing setuptools, pip, wheel...done.
% venv/bin/pip install -U pip setuptools
Downloading/unpacking pip from https://pypi.python.org/...
...
Successfully installed pip setuptools
% venv/bin/pip install tahoe-lafs
Collecting tahoe-lafs
...
Installing collected packages: ...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.15.0
foolscap: ...
%
On OS-X, instead of ``pip install tahoe-lafs``, use this command to take
advantage of the hosted pre-compiled wheels::
venv/bin/pip install --find-links=https://tahoe-lafs.org/deps tahoe-lafs
Install From a Source Tarball
-----------------------------
@ -182,13 +178,13 @@ You can also install directly from the source tarball URL::
% virtualenv venv
New python executable in ~/venv/bin/python2.7
Installing setuptools, pip, wheel...done.
% venv/bin/pip install https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2
Collecting https://tahoe-lafs.org/downloads/tahoe-lafs-1.15.0.tar.bz2
...
Installing collected packages: ...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.15.0
...
@ -213,16 +209,16 @@ with the ``--editable`` flag. You should also use the ``[test]`` extra to get
the additional libraries needed to run the unit tests::
% git clone https://github.com/tahoe-lafs/tahoe-lafs.git
% cd tahoe-lafs
% virtualenv venv
% venv/bin/pip install --editable .[test]
Obtaining file::~/tahoe-lafs
...
Successfully installed ...
% venv/bin/tahoe --version
tahoe-lafs: 1.15.0
...
@ -282,7 +278,7 @@ result in a "all tests passed" mesage::
test_missing_signature ... [OK]
...
Ran 1186 tests in 423.179s
PASSED (skips=7, expectedFailures=3, successes=1176)
__________________________ summary ___________________________________
py27: commands succeeded

0
docs/README.md → docs/README.txt
View File

BIN
docs/_static/media/image1.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 111 KiB

BIN
docs/_static/media/image2.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.3 KiB

4
docs/about.rst
View File

@ -67,12 +67,12 @@ Here's how it works:
A "storage grid" is made up of a number of storage servers. A storage server
has direct attached storage (typically one or more hard disks). A "gateway"
communicates with storage nodes, and uses them to provide access to the
grid over protocols such as HTTP(S), SFTP or FTP.
grid over protocols such as HTTP(S) and SFTP.
Note that you can find "client" used to refer to gateway nodes (which act as
a client to storage servers), and also to processes or programs connecting to
a gateway node and performing operations on the grid -- for example, a CLI
command, Web browser, SFTP client, or FTP client.
command, Web browser, or SFTP client.
Users do not rely on storage servers to provide *confidentiality* nor
*integrity* for their data -- instead all of the data is encrypted and

3
docs/anonymity-configuration.rst
View File

@ -273,7 +273,7 @@ Then, do the following:
[connections]
tcp = tor
* Launch the Tahoe server with ``tahoe start $NODEDIR``
* Launch the Tahoe server with ``tahoe run $NODEDIR``
The ``tub.port`` section will cause the Tahoe server to listen on PORT, but
bind the listening socket to the loopback interface, which is not reachable
@ -435,4 +435,3 @@ It is therefore important that your I2P router is sharing bandwidth with other
routers, so that you can give back as you use I2P. This will never impair the
performance of your Tahoe-LAFS node, because your I2P router will always
prioritize your own traffic.

6
docs/backdoors.rst
View File

@ -64,3 +64,9 @@ Peter Secor
Shawn Willden
Terrell Russell
Jean-Paul Calderone
meejah
Sajith Sasidharan

4
docs/conf.py
View File

@ -28,7 +28,7 @@ import os
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []
extensions = ['recommonmark']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@ -36,7 +36,7 @@ templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
source_suffix = ['.rst', '.md']
# The encoding of source files.
#source_encoding = 'utf-8-sig'

68
docs/configuration.rst
View File

@ -75,13 +75,12 @@ The item descriptions below use the following types:
Node Types
==========
A node can be a client/server, an introducer, or a statistics gatherer.
A node can be a client/server or an introducer.
Client/server nodes provide one or more of the following services:
* web-API service
* SFTP service
* FTP service
* helper service
* storage service.
@ -365,7 +364,7 @@ set the ``tub.location`` option described below.
also generally reduced when operating in private mode.
When False, any of the following configuration problems will cause
``tahoe start`` to throw a PrivacyError instead of starting the node:
``tahoe run`` to throw a PrivacyError instead of starting the node:
* ``[node] tub.location`` contains any ``tcp:`` hints
@ -398,13 +397,13 @@ This section controls *when* Tor and I2P are used. The ``[tor]`` and
``[i2p]`` sections (described later) control *how* Tor/I2P connections are
managed.
All Tahoe nodes need to make a connection to the Introducer; the ``[client]
introducer.furl`` setting (described below) indicates where the Introducer
lives. Tahoe client nodes must also make connections to storage servers:
these targets are specified in announcements that come from the Introducer.
Both are expressed as FURLs (a Foolscap URL), which include a list of
"connection hints". Each connection hint describes one (of perhaps many)
network endpoints where the service might live.
All Tahoe nodes need to make a connection to the Introducer; the
``private/introducers.yaml`` file (described below) configures where one or more
Introducers live. Tahoe client nodes must also make connections to storage
servers: these targets are specified in announcements that come from the
Introducer. Both are expressed as FURLs (a Foolscap URL), which include a
list of "connection hints". Each connection hint describes one (of perhaps
many) network endpoints where the service might live.
Connection hints include a type, and look like:
@ -580,6 +579,8 @@ Client Configuration
``introducer.furl = (FURL string, mandatory)``
DEPRECATED. See :ref:`introducer-definitions`.
This FURL tells the client how to connect to the introducer. Each
Tahoe-LAFS grid is defined by an introducer. The introducer's FURL is
created by the introducer node and written into its private base
@ -591,11 +592,6 @@ Client Configuration
If provided, the node will attempt to connect to and use the given helper
for uploads. See :doc:`helper` for details.
``stats_gatherer.furl = (FURL string, optional)``
If provided, the node will connect to the given stats gatherer and
provide it with operational statistics.
``shares.needed = (int, optional) aka "k", default 3``
``shares.total = (int, optional) aka "N", N >= k, default 10``
@ -711,12 +707,12 @@ CLI
file store, uploading/downloading files, and creating/running Tahoe
nodes. See :doc:`frontends/CLI` for details.
SFTP, FTP
SFTP
Tahoe can also run both SFTP and FTP servers, and map a username/password
Tahoe can also run SFTP servers, and map a username/password
pair to a top-level Tahoe directory. See :doc:`frontends/FTP-and-SFTP`
for instructions on configuring these services, and the ``[sftpd]`` and
``[ftpd]`` sections of ``tahoe.cfg``.
for instructions on configuring this service, and the ``[sftpd]``
section of ``tahoe.cfg``.
Storage Server Configuration
@ -909,11 +905,6 @@ This section describes these other files.
This file is used to construct an introducer, and is created by the
"``tahoe create-introducer``" command.
``tahoe-stats-gatherer.tac``
This file is used to construct a statistics gatherer, and is created by the
"``tahoe create-stats-gatherer``" command.
``private/control.furl``
This file contains a FURL that provides access to a control port on the
@ -965,29 +956,28 @@ This section describes these other files.
with as many people as possible, put the empty string (so that
``private/convergence`` is a zero-length file).
Additional Introducer Definitions
=================================
.. _introducer-definitions:
The ``private/introducers.yaml`` file defines additional Introducers. The
first introducer is defined in ``tahoe.cfg``, in ``[client]
introducer.furl``. To use two or more Introducers, choose a locally-unique
"petname" for each one, then define their FURLs in
``private/introducers.yaml`` like this::
Introducer Definitions
======================
The ``private/introducers.yaml`` file defines Introducers.
Choose a locally-unique "petname" for each one then define their FURLs in ``private/introducers.yaml`` like this::
introducers:
petname2:
furl: FURL2
furl: "FURL2"
petname3:
furl: FURL3
furl: "FURL3"
Servers will announce themselves to all configured introducers. Clients will
merge the announcements they receive from all introducers. Nothing will
re-broadcast an announcement (i.e. telling introducer 2 about something you
heard from introducer 1).
If you omit the introducer definitions from both ``tahoe.cfg`` and
``introducers.yaml``, the node will not use an Introducer at all. Such
"introducerless" clients must be configured with static servers (described
If you omit the introducer definitions from ``introducers.yaml``,
the node will not use an Introducer at all.
Such "introducerless" clients must be configured with static servers (described
below), or they will not be able to upload and download files.
Static Server Definitions
@ -1152,7 +1142,6 @@ a legal one.
timeout.disconnect = 1800
[client]
introducer.furl = pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo
helper.furl = pb://ggti5ssoklj4y7eok5c3xkmj@tcp:helper.tahoe.example:7054/kk8lhr
[storage]
@ -1163,6 +1152,11 @@ a legal one.
[helper]
enabled = True
To be introduced to storage servers, here is a sample ``private/introducers.yaml`` which can be used in conjunction::
introducers:
examplegrid:
furl: "pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo"
Old Configuration Files
=======================

1
docs/contributing.rst Normal file
View File

@ -0,0 +1 @@
.. include:: ../.github/CONTRIBUTING.rst

67
docs/developer-guide.rst
View File

@ -5,23 +5,17 @@ Developer Guide
Pre-commit Checks
-----------------
This project is configured for use with `pre-commit`_ to install `VCS/git hooks`_ which
perform some static code analysis checks and other code checks to catch common errors
before each commit and to run the full self-test suite to find less obvious regressions
before each push to a remote.
This project is configured for use with `pre-commit`_ to install `VCS/git hooks`_ which perform some static code analysis checks and other code checks to catch common errors.
These hooks can be configured to run before commits or pushes
For example::
tahoe-lafs $ make install-vcs-hooks
...
+ ./.tox//py36/bin/pre-commit install --hook-type pre-commit
pre-commit installed at .git/hooks/pre-commit
+ ./.tox//py36/bin/pre-commit install --hook-type pre-push
tahoe-lafs $ pre-commit install --hook-type pre-push
pre-commit installed at .git/hooks/pre-push
tahoe-lafs $ python -c "import pathlib; pathlib.Path('src/allmydata/tabbed.py').write_text('def foo():\\n\\tpass\\n')"
tahoe-lafs $ git add src/allmydata/tabbed.py
tahoe-lafs $ echo "undefined" > src/allmydata/undefined_name.py
tahoe-lafs $ git add src/allmydata/undefined_name.py
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
...
tahoe-lafs $ git push
codechecks...............................................................Failed
- hook id: codechecks
- exit code: 1
@ -30,58 +24,17 @@ For example::
codechecks inst-nodeps: ...
codechecks installed: ...
codechecks run-test-pre: PYTHONHASHSEED='...'
codechecks run-test: commands[0] | flake8 src static misc setup.py
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
codechecks run-test: commands[0] | flake8 src/allmydata/undefined_name.py
src/allmydata/undefined_name.py:1:1: F821 undefined name 'undefined'
ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src/allmydata/undefined_name.py (exited with code 1)
___________________________________ summary ____________________________________
ERROR: codechecks: commands failed
...
To uninstall::
tahoe-lafs $ make uninstall-vcs-hooks
...
+ ./.tox/py36/bin/pre-commit uninstall
pre-commit uninstalled
+ ./.tox/py36/bin/pre-commit uninstall -t pre-push
tahoe-lafs $ pre-commit uninstall --hook-type pre-push
pre-push uninstalled
Note that running the full self-test suite takes several minutes so expect pushing to
take some time. If you can't or don't want to wait for the hooks in some cases, use the
``--no-verify`` option to ``$ git commit ...`` or ``$ git push ...``. Alternatively,
see the `pre-commit`_ documentation and CLI help output and use the committed
`pre-commit configuration`_ as a starting point to write a local, uncommitted
``../.pre-commit-config.local.yaml`` configuration to use instead. For example::
tahoe-lafs $ ./.tox/py36/bin/pre-commit --help
tahoe-lafs $ ./.tox/py36/bin/pre-commit instll --help
tahoe-lafs $ cp "./.pre-commit-config.yaml" "./.pre-commit-config.local.yaml"
tahoe-lafs $ editor "./.pre-commit-config.local.yaml"
...
tahoe-lafs $ ./.tox/py36/bin/pre-commit install -c "./.pre-commit-config.local.yaml" -t pre-push
pre-commit installed at .git/hooks/pre-push
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
[3398.pre-commit 29f8f43d2] Add a file that violates flake8
1 file changed, 2 insertions(+)
create mode 100644 src/allmydata/tabbed.py
tahoe-lafs $ git push
...
codechecks...............................................................Failed
- hook id: codechecks
- exit code: 1
GLOB sdist-make: ./tahoe-lafs/setup.py
codechecks inst-nodeps: ...
codechecks installed: ...
codechecks run-test-pre: PYTHONHASHSEED='...'
codechecks run-test: commands[0] | flake8 src static misc setup.py
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
___________________________________ summary ____________________________________
ERROR: codechecks: commands failed
...
error: failed to push some refs to 'github.com:jaraco/tahoe-lafs.git'
.. _`pre-commit`: https://pre-commit.com

24
docs/frontends/CLI.rst
View File

@ -85,7 +85,7 @@ Node Management
"``tahoe create-node [NODEDIR]``" is the basic make-a-new-node
command. It creates a new directory and populates it with files that
will allow the "``tahoe start``" and related commands to use it later
will allow the "``tahoe run``" and related commands to use it later
on. ``tahoe create-node`` creates nodes that have client functionality
(upload/download files), web API services (controlled by the
'[node]web.port' configuration), and storage services (unless
@ -94,8 +94,7 @@ on. ``tahoe create-node`` creates nodes that have client functionality
NODEDIR defaults to ``~/.tahoe/`` , and newly-created nodes default to
publishing a web server on port 3456 (limited to the loopback interface, at
127.0.0.1, to restrict access to other programs on the same host). All of the
other "``tahoe``" subcommands use corresponding defaults (with the exception
that "``tahoe run``" defaults to running a node in the current directory).
other "``tahoe``" subcommands use corresponding defaults.
"``tahoe create-client [NODEDIR]``" creates a node with no storage service.
That is, it behaves like "``tahoe create-node --no-storage [NODEDIR]``".
@ -117,25 +116,6 @@ the same way on all platforms and logs to stdout. If you want to run
the process as a daemon, it is recommended that you use your favourite
daemonization tool.
The now-deprecated "``tahoe start [NODEDIR]``" command will launch a
previously-created node. It will launch the node into the background
using ``tahoe daemonize`` (and internal-only command, not for user
use). On some platforms (including Windows) this command is unable to
run a daemon in the background; in that case it behaves in the same
way as "``tahoe run``". ``tahoe start`` also monitors the logs for up
to 5 seconds looking for either a succesful startup message or for
early failure messages and produces an appropriate exit code. You are
encouraged to use ``tahoe run`` along with your favourite
daemonization tool instead of this. ``tahoe start`` is maintained for
backwards compatibility of users already using it; new scripts should
depend on ``tahoe run``.
"``tahoe stop [NODEDIR]``" will shut down a running node. "``tahoe
restart [NODEDIR]``" will stop and then restart a running
node. Similar to above, you should use ``tahoe run`` instead alongside
your favourite daemonization tool.
File Store Manipulation
=======================

81
docs/frontends/FTP-and-SFTP.rst
View File

@ -1,22 +1,21 @@
.. -*- coding: utf-8-with-signature -*-
=================================
Tahoe-LAFS SFTP and FTP Frontends
=================================
========================
Tahoe-LAFS SFTP Frontend
========================
1. `SFTP/FTP Background`_
1. `SFTP Background`_
2. `Tahoe-LAFS Support`_
3. `Creating an Account File`_
4. `Running An Account Server (accounts.url)`_
5. `Configuring SFTP Access`_
6. `Configuring FTP Access`_
7. `Dependencies`_
8. `Immutable and Mutable Files`_
9. `Known Issues`_
6. `Dependencies`_
7. `Immutable and Mutable Files`_
8. `Known Issues`_
SFTP/FTP Background
===================
SFTP Background
===============
FTP is the venerable internet file-transfer protocol, first developed in
1971. The FTP server usually listens on port 21. A separate connection is
@ -33,20 +32,18 @@ Both FTP and SFTP were developed assuming a UNIX-like server, with accounts
and passwords, octal file modes (user/group/other, read/write/execute), and
ctime/mtime timestamps.
We recommend SFTP over FTP, because the protocol is better, and the server
implementation in Tahoe-LAFS is more complete. See `Known Issues`_, below,
for details.
Previous versions of Tahoe-LAFS supported FTP, but now only the superior SFTP
frontend is supported. See `Known Issues`_, below, for details on the
limitations of SFTP.
Tahoe-LAFS Support
==================
All Tahoe-LAFS client nodes can run a frontend SFTP server, allowing regular
SFTP clients (like ``/usr/bin/sftp``, the ``sshfs`` FUSE plugin, and many
others) to access the file store. They can also run an FTP server, so FTP
clients (like ``/usr/bin/ftp``, ``ncftp``, and others) can too. These
frontends sit at the same level as the web-API interface.
others) to access the file store.
Since Tahoe-LAFS does not use user accounts or passwords, the SFTP/FTP
Since Tahoe-LAFS does not use user accounts or passwords, the SFTP
servers must be configured with a way to first authenticate a user (confirm
that a prospective client has a legitimate claim to whatever authorities we
might grant a particular user), and second to decide what directory cap
@ -173,39 +170,6 @@ clients and with the sshfs filesystem, see wiki:SftpFrontend_
.. _wiki:SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend
Configuring FTP Access
======================
To enable the FTP server with an accounts file, add the following lines to
the BASEDIR/tahoe.cfg file::
[ftpd]
enabled = true
port = tcp:8021:interface=127.0.0.1
accounts.file = private/accounts
The FTP server will listen on the given port number and on the loopback
interface only. The "accounts.file" pathname will be interpreted relative to
the node's BASEDIR.
To enable the FTP server with an account server instead, provide the URL of
that server in an "accounts.url" directive::
[ftpd]
enabled = true
port = tcp:8021:interface=127.0.0.1
accounts.url = https://example.com/login
You can provide both accounts.file and accounts.url, although it probably
isn't very useful except for testing.
FTP provides no security, and so your password or caps could be eavesdropped
if you connect to the FTP server remotely. The examples above include
":interface=127.0.0.1" in the "port" option, which causes the server to only
accept connections from localhost.
Public key authentication is not supported for FTP.
Dependencies
============
@ -216,7 +180,7 @@ separately: debian puts it in the "python-twisted-conch" package.
Immutable and Mutable Files
===========================
All files created via SFTP (and FTP) are immutable files. However, files can
All files created via SFTP are immutable files. However, files can
only be created in writeable directories, which allows the directory entry to
be relinked to a different file. Normally, when the path of an immutable file
is opened for writing by SFTP, the directory entry is relinked to another
@ -256,18 +220,3 @@ See also wiki:SftpFrontend_.
.. _ticket #1059: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1059
.. _ticket #1089: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1089
Known Issues in the FTP Frontend
--------------------------------
Mutable files are not supported by the FTP frontend (`ticket #680`_).
Non-ASCII filenames are not supported by FTP (`ticket #682`_).
The FTP frontend sometimes fails to report errors, for example if an upload
fails because it does meet the "servers of happiness" threshold (`ticket
#1081`_).
.. _ticket #680: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/680
.. _ticket #682: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/682
.. _ticket #1081: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1081

13
docs/frontends/webapi.rst
View File

@ -2032,10 +2032,11 @@ potential for surprises when the file store structure is changed.
Tahoe-LAFS provides a mutable file store, but the ways that the store can
change are limited. The only things that can change are:
* the mapping from child names to child objects inside mutable directories
(by adding a new child, removing an existing child, or changing an
existing child to point to a different object)
* the contents of mutable files
* the mapping from child names to child objects inside mutable directories
(by adding a new child, removing an existing child, or changing an
existing child to point to a different object)
* the contents of mutable files
Obviously if you query for information about the file store and then act
to change it (such as by getting a listing of the contents of a mutable
@ -2145,7 +2146,7 @@ you could do the following::
tahoe debug dump-cap URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861
-> storage index: whpepioyrnff7orecjolvbudeu
echo "whpepioyrnff7orecjolvbudeu my puppy told me to" >>$NODEDIR/access.blacklist
tahoe restart $NODEDIR
# ... restart the node to re-read configuration ...
tahoe get URI:CHK:n7r3m6wmomelk4sep3kw5cvduq:os7ijw5c3maek7pg65e5254k2fzjflavtpejjyhshpsxuqzhcwwq:3:20:14861
-> error, 403 Access Prohibited: my puppy told me to
@ -2157,7 +2158,7 @@ When modifying the file, be careful to update it atomically, otherwise a
request may arrive while the file is only halfway written, and the partial
file may be incorrectly parsed.
The blacklist is applied to all access paths (including SFTP, FTP, and CLI
The blacklist is applied to all access paths (including SFTP and CLI
operations), not just the web-API. The blacklist also applies to directories.
If a directory is blacklisted, the gateway will refuse access to both that
directory and any child files/directories underneath it, when accessed via

2
docs/helper.rst
View File

@ -122,7 +122,7 @@ Who should consider using a Helper?
* clients who experience problems with TCP connection fairness: if other
programs or machines in the same home are getting less than their fair
share of upload bandwidth. If the connection is being shared fairly, then
a Tahoe upload that is happening at the same time as a single FTP upload
a Tahoe upload that is happening at the same time as a single SFTP upload
should get half the bandwidth.
* clients who have been given the helper.furl by someone who is running a
Helper and is willing to let them use it

11
docs/historical/configuration.rst
View File

@ -20,10 +20,10 @@ Config setting File Comment
``[node]log_gatherer.furl`` ``BASEDIR/log_gatherer.furl`` (one per line)
``[node]timeout.keepalive`` ``BASEDIR/keepalive_timeout``
``[node]timeout.disconnect`` ``BASEDIR/disconnect_timeout``
``[client]introducer.furl`` ``BASEDIR/introducer.furl``
``BASEDIR/introducer.furl`` ``BASEDIR/private/introducers.yaml``
``[client]helper.furl`` ``BASEDIR/helper.furl``
``[client]key_generator.furl`` ``BASEDIR/key_generator.furl``
``[client]stats_gatherer.furl`` ``BASEDIR/stats_gatherer.furl``
``BASEDIR/stats_gatherer.furl`` Stats gatherer has been removed.
``[storage]enabled`` ``BASEDIR/no_storage`` (``False`` if ``no_storage`` exists)
``[storage]readonly`` ``BASEDIR/readonly_storage`` (``True`` if ``readonly_storage`` exists)
``[storage]sizelimit`` ``BASEDIR/sizelimit``
@ -47,3 +47,10 @@ the now (since Tahoe-LAFS v1.3.0) unsupported
addresses specified in ``advertised_ip_addresses`` were used in
addition to any that were automatically discovered), whereas the new
``tahoe.cfg`` directive is not (``tub.location`` is used verbatim).
The stats gatherer has been broken at least since Tahoe-LAFS v1.13.0.
The (broken) functionality of ``[client]stats_gatherer.furl`` (which
was previously in ``BASEDIR/stats_gatherer.furl``), is scheduled to be
completely removed after Tahoe-LAFS v1.15.0. After that point, if
your configuration contains a ``[client]stats_gatherer.furl``, your
node will refuse to start.

110
docs/how_to_make_a_tahoe-lafs_release.org
View File

@ -1,110 +0,0 @@
How to Make a Tahoe-LAFS Release
Any developer with push priveleges can do most of these steps, but a
"Release Maintainer" is required for some signing operations -- these
steps are marked with (Release Maintainer). Currently, the following
people are Release Maintainers:
- Brian Warner (https://github.com/warner)
* select features/PRs for new release [0/2]
- [ ] made sure they are tagged/labeled
- [ ] merged all release PRs
* basic quality checks [0/3]
- [ ] all travis CI checks pass
- [ ] all appveyor checks pass
- [ ] all buildbot workers pass their checks
* freeze master branch [0/1]
- [ ] announced the freeze of the master branch on IRC (i.e. non-release PRs won't be merged until after release)
* sync documentation [0/7]
- [ ] NEWS.rst: (run "tox -e news")
- [ ] added final release name and date to top-most item in NEWS.rst
- [ ] updated relnotes.txt (change next, last versions; summarize NEWS)
- [ ] updated CREDITS
- [ ] updated docs/known_issues.rst
- [ ] docs/INSTALL.rst only points to current tahoe-lafs-X.Y.Z.tar.gz source code file
- [ ] updated https://tahoe-lafs.org/hacktahoelafs/
* sign + build the tag [0/8]
- [ ] code passes all checks / tests (i.e. all CI is green)
- [ ] documentation is ready (see above)
- [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
- [ ] build code locally:
tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
- [ ] created tarballs (they'll be in dist/ for later comparison)
tox -e tarballs
- [ ] release version is reporting itself as intended version
ls dist/
- [ ] 'git pull' doesn't pull anything
- [ ] pushed tag to trigger buildslaves
git push official master TAGNAME
- [ ] confirmed Dockerhub built successfully:
https://hub.docker.com/r/tahoelafs/base/builds/
* sign the release artifacts [0/8]
- [ ] (Release Maintainer): pushed signed tag (should trigger Buildbot builders)
- [ ] Buildbot workers built all artifacts successfully
- [ ] downloaded upstream tarballs+wheels
- [ ] announce on IRC that master is unlocked
- [ ] compared upstream tarballs+wheels against local copies
- [ ] (Release Maintainer): signed each upstream artifacts with "gpg -ba -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A FILE"
- [ ] added to relnotes.txt: [0/3]
- [ ] prefix with SHA256 of tarballs
- [ ] release pubkey
- [ ] git revision hash
- [ ] GPG-signed the release email with release key (write to
relnotes.txt.asc) Ideally this is a Release Maintainer, but could
be any developer
* publish release artifacts [0/9]
- [ ] uploaded to PyPI via: twine upload dist/*
- [ ] uploaded *.asc to org ~source/downloads/
- [ ] test install works properly: pip install tahoe-lafs
- [ ] copied the release tarballs and signatures to tahoe-lafs.org: ~source/downloads/
- [ ] moved old release out of ~source/downloads (to downloads/old/?)
- [ ] ensured readthedocs.org updated
- [ ] uploaded wheels to https://tahoe-lafs.org/deps/
- [ ] uploaded release to https://github.com/tahoe-lafs/tahoe-lafs/releases
* check release downloads [0/]
- [ ] test PyPI via: pip install tahoe-lafs
- [ ] https://github.com/tahoe-lafs/tahoe-lafs/releases
- [ ] https://tahoe-lafs.org/downloads/
- [ ] https://tahoe-lafs.org/deps/
* document release in trac [0/]
- [ ] closed the Milestone on the trac Roadmap
* unfreeze master branch [0/]
- [ ] announced on IRC that new PRs will be looked at/merged
* announce new release [0/]
- [ ] sent release email and relnotes.txt.asc to tahoe-announce@tahoe-lafs.org
- [ ] sent release email and relnotes.txt.asc to tahoe-dev@tahoe-lafs.org
- [ ] updated Wiki front page: version on download link, News column
- [ ] updated Wiki "Doc": parade of release notes (with rev of NEWS.rst)
- [ ] make an "announcement of new release" on freshmeat (XXX still a thing?)
- [ ] make an "announcement of new release" on launchpad
- [ ] tweeted as @tahoelafs
- [ ] emailed relnotes.txt.asc to below listed mailing-lists/organizations
- [ ] also announce release to (trimmed from previous version of this doc):
- twisted-python@twistedmatrix.com
- liberationtech@lists.stanford.edu
- lwn@lwn.net
- p2p-hackers@lists.zooko.com
- python-list@python.org
- http://listcultures.org/pipermail/p2presearch_listcultures.org/
- cryptopp-users@googlegroups.com
- (others?)

3
docs/index.rst
View File

@ -23,8 +23,9 @@ Contents:
frontends/download-status
known_issues
../.github/CONTRIBUTING
contributing
CODE_OF_CONDUCT
release-checklist
servers
helper

6
docs/known_issues.rst
View File

@ -23,7 +23,7 @@ Known Issues in Tahoe-LAFS v1.10.3, released 30-Mar-2016
* `Disclosure of file through embedded hyperlinks or JavaScript in that file`_
* `Command-line arguments are leaked to other local users`_
* `Capabilities may be leaked to web browser phishing filter / "safe browsing" servers`_
* `Known issues in the FTP and SFTP frontends`_
* `Known issues in the SFTP frontend`_
* `Traffic analysis based on sizes of files/directories, storage indices, and timing`_
* `Privacy leak via Google Chart API link in map-update timing web page`_
@ -213,8 +213,8 @@ To disable the filter in Chrome:
----
Known issues in the FTP and SFTP frontends
------------------------------------------
Known issues in the SFTP frontend
---------------------------------
These are documented in :doc:`frontends/FTP-and-SFTP` and on `the
SftpFrontend page`_ on the wiki.

9
docs/logging.rst
View File

@ -128,10 +128,9 @@ provided in ``misc/incident-gatherer/support_classifiers.py`` . There is
roughly one category for each ``log.WEIRD``-or-higher level event in the
Tahoe source code.
The incident gatherer is created with the "``flogtool
create-incident-gatherer WORKDIR``" command, and started with "``tahoe
start``". The generated "``gatherer.tac``" file should be modified to add
classifier functions.
The incident gatherer is created with the "``flogtool create-incident-gatherer
WORKDIR``" command, and started with "``tahoe run``". The generated
"``gatherer.tac``" file should be modified to add classifier functions.
The incident gatherer writes incident names (which are simply the relative
pathname of the ``incident-\*.flog.bz2`` file) into ``classified/CATEGORY``.
@ -175,7 +174,7 @@ things that happened on multiple machines (such as comparing a client node
making a request with the storage servers that respond to that request).
Create the Log Gatherer with the "``flogtool create-gatherer WORKDIR``"
command, and start it with "``tahoe start``". Then copy the contents of the
command, and start it with "``twistd -ny gatherer.tac``". Then copy the contents of the
``log_gatherer.furl`` file it creates into the ``BASEDIR/tahoe.cfg`` file
(under the key ``log_gatherer.furl`` of the section ``[node]``) of all nodes
that should be sending it log events. (See :doc:`configuration`)

3
docs/man/man1/tahoe.1
View File

@ -45,9 +45,6 @@ Create a client node (with storage initially disabled).
.TP
.B \f[B]create-introducer\f[]
Create an introducer node.
.TP
.B \f[B]create-stats-gatherer\f[]
Create a stats-gatherer service.
.SS OPTIONS
.TP
.B \f[B]-C,\ --basedir=\f[]

218
docs/release-checklist.rst Normal file
View File

@ -0,0 +1,218 @@
=================
Release Checklist
=================
These instructions were produced while making the 1.15.0 release. They
are based on the original instructions (in old revisions in the file
`docs/how_to_make_a_tahoe-lafs_release.org`).
Any contributor can do the first part of the release preparation. Only
certain contributors can perform other parts. These are the two main
sections of this checklist (and could be done by different people).
A final section describes how to announce the release.
Any Contributor
---------------
Anyone who can create normal PRs should be able to complete this
portion of the release process.
Prepare for the Release
```````````````````````
The `master` branch should always be releasable.
It may be worth asking (on IRC or mailing-ist) if anything will be
merged imminently (for example, "I will prepare a release this coming
Tuesday if you want to get anything in").
- Create a ticket for the release in Trac
- Ticket number needed in next section
Create Branch and Apply Updates
```````````````````````````````
- Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`)
- run `tox -e news` to produce a new NEWS.txt file (this does a commit)
- create the news for the release
- newsfragments/<ticket number>.minor
- commit it
- manually fix NEWS.txt
- proper title for latest release ("Release 1.15.0" instead of "Release ...post1432")
- double-check date (maybe release will be in the future)
- spot-check the release notes (these come from the newsfragments
files though so don't do heavy editing)
- commit these changes
- update "relnotes.txt"
- update all mentions of 1.14.0 -> 1.15.0
- update "previous release" statement and date
- summarize major changes
- commit it
- update "CREDITS"
- are there any new contributors in this release?
- one way: git log release-1.14.0.. | grep Author | sort | uniq
- commit it
- update "docs/known_issues.rst" if appropriate
- update "docs/INSTALL.rst" references to the new release
- Push the branch to github
- Create a (draft) PR; this should trigger CI (note that github
doesn't let you create a PR without some changes on the branch so
running + committing the NEWS.txt file achieves that without changing
any code)
- Confirm CI runs successfully on all platforms
Create Release Candidate
````````````````````````
Before "officially" tagging any release, we will make a
release-candidate available. So there will be at least 1.15.0rc0 (for
example). If there are any problems, an rc1 or rc2 etc may also be
released. Anyone can sign these releases (ideally they'd be signed
"officially" as well, but it's better to get them out than to wait for
that).
Typically expert users will be the ones testing release candidates and
they will need to evaluate which contributors' signatures they trust.
- (all steps above are completed)
- sign the release
- git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0
- (replace the key-id above with your own)
- build all code locally
- these should all pass:
- tox -e py27,codechecks,docs,integration
- these can fail (ideally they should not of course):
- tox -e deprecations,upcoming-deprecations
- build tarballs
- tox -e tarballs
- confirm it at least exists:
- ls dist/ | grep 1.15.0rc0
- inspect and test the tarballs
- install each in a fresh virtualenv
- run `tahoe` command
- when satisfied, sign the tarballs:
- gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl
- gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2
- gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz
- gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip
Privileged Contributor
-----------------------
Steps in this portion require special access to keys or
infrastructure. For example, **access to tahoe-lafs.org** to upload
binaries or edit HTML.
Hack Tahoe-LAFS
```````````````
Did anyone contribute a hack since the last release? If so, then
https://tahoe-lafs.org/hacktahoelafs/ needs to be updated.
Upload Artifacts
````````````````
Any release-candidate or actual release plus signature (.asc file)
need to be uploaded to https://tahoe-lafs.org in `~source/downloads`
- secure-copy all release artifacts to the download area on the
tahoe-lafs.org host machine. `~source/downloads` on there maps to
https://tahoe-lafs.org/downloads/ on the Web.
- scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads
- the following developers have access to do this:
- exarkun
- meejah
- warner
For the actual release, the tarball and signature files need to be
uploaded to PyPI as well.
- how to do this?
- (original guide says only `twine upload dist/*`)
- the following developers have access to do this:
- warner
- exarkun (partial?)
- meejah (partial?)
Announcing the Release Candidate
````````````````````````````````
The release-candidate should be announced by posting to the
mailing-list (tahoe-dev@tahoe-lafs.org). For example:
https://tahoe-lafs.org/pipermail/tahoe-dev/2020-October/009995.html
Is The Release Done Yet?
````````````````````````
If anyone reports a problem with a release-candidate then a new
release-candidate should be made once a fix has been merged to
master. Repeat the above instructions with `rc1` or `rc2` or whatever
is appropriate.
Once a release-candidate has marinated for some time then it can be
made into a the actual release.
XXX Write this section when doing 1.15.0 actual release
(In general, this means dropping the "rcX" part of the release and the
tag, uploading those artifacts, uploading to PyPI, ... )
Announcing the Release
----------------------
mailing-lists
`````````````
A new Tahoe release is traditionally announced on our mailing-list
(tahoe-dev@tahoe-lafs.org). The former version of these instructions
also announced the release on the following other lists:
- tahoe-announce@tahoe-lafs.org
- twisted-python@twistedmatrix.com
- liberationtech@lists.stanford.edu
- lwn@lwn.net
- p2p-hackers@lists.zooko.com
- python-list@python.org
- http://listcultures.org/pipermail/p2presearch_listcultures.org/
- cryptopp-users@googlegroups.com
wiki
````
Edit the "News" section of the front page of https://tahoe-lafs.org
with a link to the mailing-list archive of the announcement message.

21
docs/running.rst
View File

@ -65,9 +65,9 @@ Running a Client
To construct a client node, run “``tahoe create-client``”, which will create
``~/.tahoe`` to be the node's base directory. Acquire the ``introducer.furl``
(see below if you are running your own introducer, or use the one from the
`TestGrid page`_), and paste it after ``introducer.furl =`` in the
``[client]`` section of ``~/.tahoe/tahoe.cfg``. Then use “``tahoe run
~/.tahoe``”. After that, the node should be off and running. The first thing
`TestGrid page`_), and write it to ``~/.tahoe/private/introducers.yaml``
(see :ref:`introducer-definitions`). Then use “``tahoe run ~/.tahoe``”.
After that, the node should be off and running. The first thing
it will do is connect to the introducer and get itself connected to all other
nodes on the grid.
@ -81,9 +81,7 @@ does not offer its disk space to other nodes. To configure other behavior,
use “``tahoe create-node``” or see :doc:`configuration`.
The “``tahoe run``” command above will run the node in the foreground.
On Unix, you can run it in the background instead by using the
“``tahoe start``” command. To stop a node started in this way, use
“``tahoe stop``”. ``tahoe --help`` gives a summary of all commands.
``tahoe --help`` gives a summary of all commands.
Running a Server or Introducer
@ -99,12 +97,10 @@ and ``--location`` arguments.
To construct an introducer, create a new base directory for it (the name
of the directory is up to you), ``cd`` into it, and run “``tahoe
create-introducer --hostname=example.net .``” (but using the hostname of
your VPS). Now run the introducer using “``tahoe start .``”. After it
your VPS). Now run the introducer using “``tahoe run .``”. After it
starts, it will write a file named ``introducer.furl`` into the
``private/`` subdirectory of that base directory. This file contains the
URL the other nodes must use in order to connect to this introducer.
(Note that “``tahoe run .``” doesn't work for introducers, this is a
known issue: `#937`_.)
You can distribute your Introducer fURL securely to new clients by using
the ``tahoe invite`` command. This will prepare some JSON to send to the
@ -211,10 +207,10 @@ create a new directory and lose the capability to it, then you cannot
access that directory ever again.
The SFTP and FTP frontends
--------------------------
The SFTP frontend
-----------------
You can access your Tahoe-LAFS grid via any SFTP_ or FTP_ client. See
You can access your Tahoe-LAFS grid via any SFTP_ client. See
:doc:`frontends/FTP-and-SFTP` for how to set this up. On most Unix
platforms, you can also use SFTP to plug Tahoe-LAFS into your computer's
local filesystem via ``sshfs``, but see the `FAQ about performance
@ -224,7 +220,6 @@ The SftpFrontend_ page on the wiki has more information about using SFTP with
Tahoe-LAFS.
.. _SFTP: https://en.wikipedia.org/wiki/SSH_file_transfer_protocol
.. _FTP: https://en.wikipedia.org/wiki/File_Transfer_Protocol
.. _FAQ about performance problems: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/FAQ#Q23_FUSE
.. _SftpFrontend: https://tahoe-lafs.org/trac/tahoe-lafs/wiki/SftpFrontend

1
docs/specifications/index.rst
View File

@ -8,6 +8,7 @@ the data formats used by Tahoe.
:maxdepth: 2
outline
url
uri
file-encoding
URI-extension

165
docs/specifications/url.rst Normal file
View File

@ -0,0 +1,165 @@
URLs
====
The goal of this document is to completely specify the construction and use of the URLs by Tahoe-LAFS for service location.
This includes, but is not limited to, the original Foolscap-based URLs.
These are not to be confused with the URI-like capabilities Tahoe-LAFS uses to refer to stored data.
An attempt is also made to outline the rationale for certain choices about these URLs.
The intended audience for this document is Tahoe-LAFS maintainers and other developers interested in interoperating with Tahoe-LAFS or these URLs.
Background
----------
Tahoe-LAFS first used Foolscap_ for network communication.
Foolscap connection setup takes as an input a Foolscap URL or a *fURL*.
A fURL includes three components:
* the base32-encoded SHA1 hash of the DER form of an x509v3 certificate
* zero or more network addresses [1]_
* an object identifier
A Foolscap client tries to connect to each network address in turn.
If a connection is established then TLS is negotiated.
The server is authenticated by matching its certificate against the hash in the fURL.
A matching certificate serves as proof that the handshaking peer is the correct server.
This serves as the process by which the client authenticates the server.
The client can then exercise further Foolscap functionality using the fURL's object identifier.
If the object identifier is an unguessable, secret string then it serves as a capability.
This unguessable identifier is sometimes called a `swiss number`_ (or swissnum).
The client's use of the swissnum is what allows the server to authorize the client.
.. _`swiss number`: http://wiki.erights.org/wiki/Swiss_number
NURLs
-----
The authentication and authorization properties of fURLs are a good fit for Tahoe-LAFS' requirements.
These are not inherently tied to the Foolscap protocol itself.
In particular they are beneficial to :doc:`../proposed/http-storage-node-protocol` which uses HTTP instead of Foolscap.
It is conceivable they will also be used with WebSockets at some point as well.
Continuing to refer to these URLs as fURLs when they are being used for other protocols may cause confusion.
Therefore,
this document coins the name **NURL** for these URLs.
This can be considered to expand to "**N**\ ew URLs" or "Authe\ **N**\ ticating URLs" or "Authorizi\ **N**\ g URLs" as the reader prefers.
The anticipated use for a **NURL** will still be to establish a TLS connection to a peer.
The protocol run over that TLS connection could be Foolscap though it is more likely to be an HTTP-based protocol (such as GBS).
Syntax
------
The EBNF for a NURL is as follows::
nurl = scheme, hash, "@", net-loc-list, "/", swiss-number, [ version1 ]
scheme = "pb://"
hash = unreserved
net-loc-list = net-loc, [ { ",", net-loc } ]
net-loc = tcp-loc | tor-loc | i2p-loc
tcp-loc = [ "tcp:" ], hostname, [ ":" port ]
tor-loc = "tor:", hostname, [ ":" port ]
i2p-loc = "i2p:", i2p-addr, [ ":" port ]
i2p-addr = { unreserved }, ".i2p"
hostname = domain | IPv4address | IPv6address
swiss-number = segment
version1 = "#v=1"
See https://tools.ietf.org/html/rfc3986#section-3.3 for the definition of ``segment``.
See https://tools.ietf.org/html/rfc2396#appendix-A for the definition of ``unreserved``.
See https://tools.ietf.org/html/draft-main-ipaddr-text-rep-02#section-3.1 for the definition of ``IPv4address``.
See https://tools.ietf.org/html/draft-main-ipaddr-text-rep-02#section-3.2 for the definition of ``IPv6address``.
See https://tools.ietf.org/html/rfc1035#section-2.3.1 for the definition of ``domain``.
Versions
--------
Though all NURLs are syntactically compatible some semantic differences are allowed.
These differences are separated into distinct versions.
Version 0
---------
A Foolscap fURL is considered the canonical definition of a version 0 NURL.
Notably,
the hash component is defined as the base32-encoded SHA1 hash of the DER form of an x509v3 certificate.
A version 0 NURL is identified by the absence of the ``v=1`` fragment.
Examples
~~~~~~~~
* ``pb://sisi4zenj7cxncgvdog7szg3yxbrnamy@tcp:127.1:34399/xphmwz6lx24rh2nxlinni``
* ``pb://2uxmzoqqimpdwowxr24q6w5ekmxcymby@localhost:47877/riqhpojvzwxujhna5szkn``
Version 1
---------
The hash component of a version 1 NURL differs in three ways from the prior version.
1. The hash function used is SHA3-224 instead of SHA1.
The security of SHA1 `continues to be eroded`_.
Contrariwise SHA3 is currently the most recent addition to the SHA family by NIST.
The 224 bit instance is chosen to keep the output short and because it offers greater collision resistance than SHA1 was thought to offer even at its inception
(prior to security research showing actual collision resistance is lower).
2. The hash is computed over the certificate's SPKI instead of the whole certificate.
This allows certificate re-generation so long as the public key remains the same.
This is useful to allow contact information to be updated or extension of validity period.
Use of an SPKI hash has also been `explored by the web community`_ during its flirtation with using it for HTTPS certificate pinning
(though this is now largely abandoned).
.. note::
*Only* the certificate's keypair is pinned by the SPKI hash.
The freedom to change every other part of the certificate is coupled with the fact that all other parts of the certificate contain arbitrary information set by the private key holder.
It is neither guaranteed nor expected that a certificate-issuing authority has validated this information.
Therefore,
*all* certificate fields should be considered within the context of the relationship identified by the SPKI hash.
3. The hash is encoded using urlsafe-base64 (without padding) instead of base32.
This provides a more compact representation and minimizes the usability impacts of switching from a 160 bit hash to a 224 bit hash.
A version 1 NURL is identified by the presence of the ``v=1`` fragment.
Though the length of the hash string (38 bytes) could also be used to differentiate it from a version 0 NURL,
there is no guarantee that this will be effective in differentiating it from future versions so this approach should not be used.
It is possible for a client to unilaterally upgrade a version 0 NURL to a version 1 NURL.
After establishing and authenticating a connection the client will have received a copy of the server's certificate.
This is sufficient to compute the new hash and rewrite the NURL to upgrade it to version 1.
This provides stronger authentication assurances for future uses but it is not required.
Examples
~~~~~~~~
* ``pb://1WUX44xKjKdpGLohmFcBNuIRN-8rlv1Iij_7rQ@tcp:127.1:34399/jhjbc3bjbhk#v=1``
* ``pb://azEu8vlRpnEeYm0DySQDeNY3Z2iJXHC_bsbaAw@localhost:47877/64i4aokv4ej#v=1``
.. _`continues to be eroded`: https://en.wikipedia.org/wiki/SHA-1#Cryptanalysis_and_validation
.. _`explored by the web community`: https://www.imperialviolet.org/2011/05/04/pinning.html
.. _Foolscap: https://github.com/warner/foolscap
.. [1] ``foolscap.furl.decode_furl`` is taken as the canonical definition of the syntax of a fURL.
The **location hints** part of the fURL,
as it is referred to in Foolscap,
is matched by the regular expression fragment ``([^/]*)``.
Since this matches the empty string,
no network addresses are required to form a fURL.
The supporting code around the regular expression also takes extra steps to allow an empty string to match here.
Open Questions
--------------
1. Should we make a hard recommendation that all certificate fields are ignored?
The system makes no guarantees about validation of these fields.
Is it just an unnecessary risk to let a user see them?
2. Should the version specifier be a query-arg-alike or a fragment-alike?
The value is only necessary on the client side which makes it similar to an HTTP URL fragment.
The current Tahoe-LAFS configuration parsing code has special handling of the fragment character (``#``) which makes it unusable.
However,
the configuration parsing code is easily changed.

89
docs/stats.rst
View File

@ -6,8 +6,7 @@ Tahoe Statistics
1. `Overview`_
2. `Statistics Categories`_
3. `Running a Tahoe Stats-Gatherer Service`_
4. `Using Munin To Graph Stats Values`_
3. `Using Munin To Graph Stats Values`_
Overview
========
@ -243,92 +242,6 @@ The currently available stats (as of release 1.6.0 or so) are described here:
the process was started. Ticket #472 indicates that .total may
sometimes be negative due to wraparound of the kernel's counter.
**stats.load_monitor.\***
When enabled, the "load monitor" continually schedules a one-second
callback, and measures how late the response is. This estimates system load
(if the system is idle, the response should be on time). This is only
enabled if a stats-gatherer is configured.
avg_load
average "load" value (seconds late) over the last minute
max_load
maximum "load" value over the last minute
Running a Tahoe Stats-Gatherer Service
======================================
The "stats-gatherer" is a simple daemon that periodically collects stats from
several tahoe nodes. It could be useful, e.g., in a production environment,
where you want to monitor dozens of storage servers from a central management
host. It merely gatherers statistics from many nodes into a single place: it
does not do any actual analysis.
The stats gatherer listens on a network port using the same Foolscap_
connection library that Tahoe clients use to connect to storage servers.
Tahoe nodes can be configured to connect to the stats gatherer and publish
their stats on a periodic basis. (In fact, what happens is that nodes connect
to the gatherer and offer it a second FURL which points back to the node's
"stats port", which the gatherer then uses to pull stats on a periodic basis.
The initial connection is flipped to allow the nodes to live behind NAT
boxes, as long as the stats-gatherer has a reachable IP address.)
.. _Foolscap: https://foolscap.lothar.com/trac
The stats-gatherer is created in the same fashion as regular tahoe client
nodes and introducer nodes. Choose a base directory for the gatherer to live
in (but do not create the directory). Choose the hostname that should be
advertised in the gatherer's FURL. Then run:
::
tahoe create-stats-gatherer --hostname=HOSTNAME $BASEDIR
and start it with "tahoe start $BASEDIR". Once running, the gatherer will
write a FURL into $BASEDIR/stats_gatherer.furl .
To configure a Tahoe client/server node to contact the stats gatherer, copy
this FURL into the node's tahoe.cfg file, in a section named "[client]",
under a key named "stats_gatherer.furl", like so:
::
[client]
stats_gatherer.furl = pb://qbo4ktl667zmtiuou6lwbjryli2brv6t@HOSTNAME:PORTNUM/wxycb4kaexzskubjnauxeoptympyf45y
or simply copy the stats_gatherer.furl file into the node's base directory
(next to the tahoe.cfg file): it will be interpreted in the same way.
When the gatherer is created, it will allocate a random unused TCP port, so
it should not conflict with anything else that you have running on that host
at that time. To explicitly control which port it uses, run the creation
command with ``--location=`` and ``--port=`` instead of ``--hostname=``. If
you use a hostname of ``example.org`` and a port number of ``1234``, then
run::
tahoe create-stats-gatherer --location=tcp:example.org:1234 --port=tcp:1234
``--location=`` is a Foolscap FURL hints string (so it can be a
comma-separated list of connection hints), and ``--port=`` is a Twisted
"server endpoint specification string", as described in :doc:`configuration`.
Once running, the stats gatherer will create a standard JSON file in
``$BASEDIR/stats.json``. Once a minute, the gatherer will pull stats
information from every connected node and write them into the file. The file
will contain a dictionary, in which node identifiers (known as "tubid"
strings) are the keys, and the values are a dict with 'timestamp',
'nickname', and 'stats' keys. d[tubid][stats] will contain the stats
dictionary as made available at http://localhost:3456/statistics?t=json . The
file will only contain the most recent update from each node.
Other tools can be built to examine these stats and render them into
something useful. For example, a tool could sum the
"storage_server.disk_avail' values from all servers to compute a
total-disk-available number for the entire grid (however, the "disk watcher"
daemon, in misc/operations_helpers/spacetime/, is better suited for this
specific task).
Using Munin To Graph Stats Values
=================================

17
docs/windows.rst
View File

@ -33,7 +33,7 @@ You can use whatever name you like for the virtualenv, but example uses
3: Use the virtualenv's ``pip`` to install the latest release of Tahoe-LAFS
into this virtualenv::
PS C:\Users\me> venv\Scripts\pip install --find-links=https://tahoe-lafs.org/deps/ tahoe-lafs
PS C:\Users\me> venv\Scripts\pip install tahoe-lafs
Collecting tahoe-lafs
...
Installing collected packages: ...
@ -69,7 +69,7 @@ The ``pip install tahoe-lafs`` command above will install the latest release
the following command (using pip from the virtualenv, from the root of your
git checkout)::
$ venv\Scripts\pip install --find-links=https://tahoe-lafs.org/deps/ .
$ venv\Scripts\pip install .
If you're planning to hack on the source code, you might want to add
``--editable`` so you won't have to re-install each time you make a change.
@ -77,12 +77,7 @@ If you're planning to hack on the source code, you might want to add
Dependencies
------------
Tahoe-LAFS depends upon several packages that use compiled C code
(such as zfec). This code must be built separately for each platform
(Windows, OS-X, and different flavors of Linux).
Pre-compiled "wheels" of all Tahoe's dependencies are hosted on the
tahoe-lafs.org website in the ``deps/`` directory. The ``--find-links=``
argument (used in the examples above) instructs ``pip`` to look at that URL
for dependencies. This should avoid the need for anything to be compiled
during the install.
Tahoe-LAFS depends upon several packages that use compiled C code (such as zfec).
This code must be built separately for each platform (Windows, OS-X, and different flavors of Linux).
Fortunately, this is now done by upstream packages for most platforms.
The result is that a C compiler is usually not required to install Tahoe-LAFS.

85
integration/conftest.py
View File

@ -7,11 +7,15 @@ from os import mkdir, listdir, environ
from os.path import join, exists
from tempfile import mkdtemp, mktemp
from functools import partial
from json import loads
from foolscap.furl import (
decode_furl,
)
from eliot import (
to_file,
log_call,
start_action,
)
from twisted.python.procutils import which
@ -30,11 +34,14 @@ from util import (
_DumpOutputProtocol,
_ProcessExitedProtocol,
_create_node,
_run_node,
_cleanup_tahoe_process,
_tahoe_runner_optional_coverage,
await_client_ready,
TahoeProcess,
cli,
_run_node,
generate_ssh_key,
block_with_timeout,
)
@ -150,7 +157,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
)
print("Waiting for flogtool to complete")
try:
pytest_twisted.blockon(flog_protocol.done)
block_with_timeout(flog_protocol.done, reactor)
except ProcessTerminated as e:
print("flogtool exited unexpectedly: {}".format(str(e)))
print("Flogtool completed")
@ -199,9 +206,8 @@ log_gatherer.furl = {log_furl}
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
protocol = _MagicTextProtocol('introducer running')
transport = _tahoe_runner_optional_coverage(
protocol,
@ -226,6 +232,16 @@ def introducer_furl(introducer, temp_dir):
print("Don't see {} yet".format(furl_fname))
sleep(.1)
furl = open(furl_fname, 'r').read()
tubID, location_hints, name = decode_furl(furl)
if not location_hints:
# If there are no location hints then nothing can ever possibly
# connect to it and the only thing that can happen next is something
# will hang or time out. So just give up right now.
raise ValueError(
"Introducer ({!r}) fURL has no location hints!".format(
introducer_furl,
),
)
return furl
@ -266,9 +282,8 @@ log_gatherer.furl = {log_furl}
with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
f.write(config)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
protocol = _MagicTextProtocol('introducer running')
transport = _tahoe_runner_optional_coverage(
protocol,
@ -283,7 +298,7 @@ log_gatherer.furl = {log_furl}
def cleanup():
try:
transport.signalProcess('TERM')
pytest_twisted.blockon(protocol.exited)
block_with_timeout(protocol.exited, reactor)
except ProcessExitedAlready:
pass
request.addfinalizer(cleanup)
@ -337,8 +352,50 @@ def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, requ
reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
web_port="tcp:9980:interface=localhost",
storage=False,
# We're going to kill this ourselves, so no need for finalizer to
# do it:
finalize=False,
)
)
await_client_ready(process)
# 1. Create a new RW directory cap:
cli(process, "create-alias", "test")
rwcap = loads(cli(process, "list-aliases", "--json"))["test"]["readwrite"]
# 2. Enable SFTP on the node:
host_ssh_key_path = join(process.node_dir, "private", "ssh_host_rsa_key")
accounts_path = join(process.node_dir, "private", "accounts")
with open(join(process.node_dir, "tahoe.cfg"), "a") as f:
f.write("""\
[sftpd]
enabled = true
port = tcp:8022:interface=127.0.0.1
host_pubkey_file = {ssh_key_path}.pub
host_privkey_file = {ssh_key_path}
accounts.file = {accounts_path}
""".format(ssh_key_path=host_ssh_key_path, accounts_path=accounts_path))
generate_ssh_key(host_ssh_key_path)
# 3. Add a SFTP access file with username/password and SSH key auth.
# The client SSH key path is typically going to be somewhere else (~/.ssh,
# typically), but for convenience sake for testing we'll put it inside node.
client_ssh_key_path = join(process.node_dir, "private", "ssh_client_rsa_key")
generate_ssh_key(client_ssh_key_path)
# Pub key format is "ssh-rsa <thekey> <username>". We want the key.
ssh_public_key = open(client_ssh_key_path + ".pub").read().strip().split()[1]
with open(accounts_path, "w") as f:
f.write("""\
alice password {rwcap}
alice2 ssh-rsa {ssh_public_key} {rwcap}
""".format(rwcap=rwcap, ssh_public_key=ssh_public_key))
# 4. Restart the node with new SFTP config.
process.kill()
pytest_twisted.blockon(_run_node(reactor, process.node_dir, request, None))
await_client_ready(process)
return process
@ -480,7 +537,13 @@ def tor_network(reactor, temp_dir, chutney, request):
path=join(chutney_dir),
env=env,
)
pytest_twisted.blockon(proto.done)
try:
block_with_timeout(proto.done, reactor)
except ProcessTerminated:
# If this doesn't exit cleanly, that's fine, that shouldn't fail
# the test suite.
pass
request.addfinalizer(cleanup)
return chut

1440
integration/install-tor.sh
View File

File diff suppressed because it is too large Load Diff

1
integration/test_servers_of_happiness.py
View File

@ -1,7 +1,6 @@
import sys
from os.path import join
from twisted.internet import task
from twisted.internet.error import ProcessTerminated
import util

162
integration/test_sftp.py Normal file
View File

@ -0,0 +1,162 @@
"""
It's possible to create/rename/delete files and directories in Tahoe-LAFS using
SFTP.
These tests use Paramiko, rather than Twisted's Conch, because:
1. It's a different implementation, so we're not testing Conch against
itself.
2. Its API is much simpler to use.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from posixpath import join
from stat import S_ISDIR
from paramiko import SSHClient
from paramiko.client import AutoAddPolicy
from paramiko.sftp_client import SFTPClient
from paramiko.ssh_exception import AuthenticationException
from paramiko.rsakey import RSAKey
import pytest
from .util import generate_ssh_key, run_in_thread
def connect_sftp(connect_args={"username": "alice", "password": "password"}):
"""Create an SFTP client."""
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy)
client.connect("localhost", port=8022, look_for_keys=False,
allow_agent=False, **connect_args)
sftp = SFTPClient.from_transport(client.get_transport())
def rmdir(path, delete_root=True):
for f in sftp.listdir_attr(path=path):
childpath = join(path, f.filename)
if S_ISDIR(f.st_mode):
rmdir(childpath)
else:
sftp.remove(childpath)
if delete_root:
sftp.rmdir(path)
# Delete any files left over from previous tests :(
rmdir("/", delete_root=False)
return sftp
@run_in_thread
def test_bad_account_password_ssh_key(alice, tmpdir):
"""
Can't login with unknown username, wrong password, or wrong SSH pub key.
"""
# Wrong password, wrong username:
for u, p in [("alice", "wrong"), ("someuser", "password")]:
with pytest.raises(AuthenticationException):
connect_sftp(connect_args={
"username": u, "password": p,
})
another_key = join(str(tmpdir), "ssh_key")
generate_ssh_key(another_key)
good_key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key"))
bad_key = RSAKey(filename=another_key)
# Wrong key:
with pytest.raises(AuthenticationException):
connect_sftp(connect_args={
"username": "alice2", "pkey": bad_key,
})
# Wrong username:
with pytest.raises(AuthenticationException):
connect_sftp(connect_args={
"username": "someoneelse", "pkey": good_key,
})
@run_in_thread
def test_ssh_key_auth(alice):
"""It's possible to login authenticating with SSH public key."""
key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key"))
sftp = connect_sftp(connect_args={
"username": "alice2", "pkey": key
})
assert sftp.listdir() == []
@run_in_thread
def test_read_write_files(alice):
"""It's possible to upload and download files."""
sftp = connect_sftp()
with sftp.file("myfile", "wb") as f:
f.write(b"abc")
f.write(b"def")
with sftp.file("myfile", "rb") as f:
assert f.read(4) == b"abcd"
assert f.read(2) == b"ef"
assert f.read(1) == b""
@run_in_thread
def test_directories(alice):
"""
It's possible to create, list directories, and create and remove files in
them.
"""
sftp = connect_sftp()
assert sftp.listdir() == []
sftp.mkdir("childdir")
assert sftp.listdir() == ["childdir"]
with sftp.file("myfile", "wb") as f:
f.write(b"abc")
assert sorted(sftp.listdir()) == ["childdir", "myfile"]
sftp.chdir("childdir")
assert sftp.listdir() == []
with sftp.file("myfile2", "wb") as f:
f.write(b"def")
assert sftp.listdir() == ["myfile2"]
sftp.chdir(None) # root
with sftp.file("childdir/myfile2", "rb") as f:
assert f.read() == b"def"
sftp.remove("myfile")
assert sftp.listdir() == ["childdir"]
sftp.rmdir("childdir")
assert sftp.listdir() == []
@run_in_thread
def test_rename(alice):
"""Directories and files can be renamed."""
sftp = connect_sftp()
sftp.mkdir("dir")
filepath = join("dir", "file")
with sftp.file(filepath, "wb") as f:
f.write(b"abc")
sftp.rename(filepath, join("dir", "file2"))
sftp.rename("dir", "dir2")
with sftp.file(join("dir2", "file2"), "rb") as f:
assert f.read() == b"abc"

12
integration/test_streaming_logs.py
View File

@ -53,7 +53,12 @@ class _StreamingLogClientProtocol(WebSocketClientProtocol):
self.factory.on_open.callback(self)
def onMessage(self, payload, isBinary):
self.on_message.callback(payload)
if self.on_message is None:
# Already did our job, ignore it
return
on_message = self.on_message
self.on_message = None
on_message.callback(payload)
def onClose(self, wasClean, code, reason):
self.on_close.callback(reason)
@ -131,10 +136,13 @@ def _test_streaming_logs(reactor, temp_dir, alice):
client.on_close = Deferred()
client.on_message = Deferred()
# Capture this now before on_message perhaps goes away.
racing = _race(client.on_close, client.on_message)
# Provoke _some_ log event.
yield treq.get(node_url)
result = yield _race(client.on_close, client.on_message)
result = yield racing
assert isinstance(result, Right)
json.loads(result.value)

36
integration/test_tor.py
View File

@ -1,21 +1,21 @@
from __future__ import print_function
import sys
import time
import shutil
from os import mkdir, unlink, listdir
from os.path import join, exists
from six.moves import StringIO
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
from twisted.internet.defer import inlineCallbacks, Deferred
from os.path import join
import pytest
import pytest_twisted
import util
from twisted.python.filepath import (
FilePath,
)
from allmydata.test.common import (
write_introducer,
)
# see "conftest.py" for the fixtures (e.g. "tor_network")
# XXX: Integration tests that involve Tor do not run reliably on
@ -73,12 +73,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
@pytest_twisted.inlineCallbacks
def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
node_dir = join(temp_dir, name)
node_dir = FilePath(temp_dir).child(name)
web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
if True:
print("creating", node_dir)
mkdir(node_dir)
print("creating", node_dir.path)
node_dir.makedirs()
proto = util._DumpOutputProtocol(None)
reactor.spawnProcess(
proto,
@ -91,12 +91,15 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
'--hide-ip',
'--tor-control-port', 'tcp:localhost:{}'.format(control_port),
'--listen', 'tor',
node_dir,
node_dir.path,
)
)
yield proto.done
with open(join(node_dir, 'tahoe.cfg'), 'w') as f:
# Which services should this client connect to?
write_introducer(node_dir, "default", introducer_furl)
with node_dir.child('tahoe.cfg').open('w') as f:
f.write('''
[node]
nickname = %(name)s
@ -112,15 +115,12 @@ onion = true
onion.private_key_file = private/tor_onion.privkey
[client]
# Which services should this client connect to?
introducer.furl = %(furl)s
shares.needed = 1
shares.happy = 1
shares.total = 2
''' % {
'name': name,
'furl': introducer_furl,
'web_port': web_port,
'log_furl': flog_gatherer,
'control_port': control_port,
@ -128,5 +128,5 @@ shares.total = 2
})
print("running")
yield util._run_node(reactor, node_dir, request, None)
yield util._run_node(reactor, node_dir.path, request, None)
print("okay, launched")

11
integration/test_web.py
View File

@ -9,20 +9,15 @@ WebAPI *should* do in every situation. It's not clear the latter
exists anywhere, however.
"""
import sys
import time
import shutil
import json
import urllib2
from os import mkdir, unlink, utime
from os.path import join, exists, getmtime
import allmydata.uri
import util
import requests
import pytest_twisted
import html5lib
from bs4 import BeautifulSoup
@ -132,12 +127,12 @@ def test_deep_stats(alice):
dircap_uri,
data={
u"t": u"upload",
u"when_done": u".",
},
files={
u"file": FILE_CONTENTS,
},
)
resp.raise_for_status()
# confirm the file is in the directory
resp = requests.get(
@ -180,6 +175,7 @@ def test_deep_stats(alice):
time.sleep(.5)
@util.run_in_thread
def test_status(alice):
"""
confirm we get something sensible from /status and the various sub-types
@ -265,7 +261,8 @@ def test_directory_deep_check(alice):
dircap_url,
params={u"t": u"json"},
)
dir_meta = json.loads(resp.content)
# Just verify it is valid JSON.
json.loads(resp.content)
# upload a file of pangrams into the directory
FILE_CONTENTS = u"Sphinx of black quartz, judge my vow.\n" * (2048*10)

96
integration/util.py
View File

@ -1,17 +1,25 @@
import sys
import time
import json
from os import mkdir
from os import mkdir, environ
from os.path import exists, join
from six.moves import StringIO
from functools import partial
from subprocess import check_output
from twisted.python.filepath import (
FilePath,
)
from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
from twisted.internet.threads import deferToThread
import requests
from paramiko.rsakey import RSAKey
from boltons.funcutils import wraps
from allmydata.util.configutil import (
get_config,
set_config,
@ -22,6 +30,12 @@ from allmydata import client
import pytest_twisted
def block_with_timeout(deferred, reactor, timeout=120):
"""Block until Deferred has result, but timeout instead of waiting forever."""
deferred.addTimeout(timeout, reactor)
return pytest_twisted.blockon(deferred)
class _ProcessExitedProtocol(ProcessProtocol):
"""
Internal helper that .callback()s on self.done when the process
@ -120,11 +134,12 @@ def _cleanup_tahoe_process(tahoe_transport, exited):
:return: After the process has exited.
"""
from twisted.internet import reactor
try:
print("signaling {} with TERM".format(tahoe_transport.pid))
tahoe_transport.signalProcess('TERM')
print("signaled, blocking on exit")
pytest_twisted.blockon(exited)
block_with_timeout(exited, reactor)
print("exited, goodbye")
except ProcessExitedAlready:
pass
@ -145,6 +160,7 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
proto,
sys.executable,
args,
env=environ,
)
@ -171,11 +187,15 @@ class TahoeProcess(object):
u"portnum",
)
def kill(self):
"""Kill the process, block until it's done."""
_cleanup_tahoe_process(self.transport, self.transport.exited)
def __str__(self):
return "<TahoeProcess in '{}'>".format(self._node_dir)
def _run_node(reactor, node_dir, request, magic_text):
def _run_node(reactor, node_dir, request, magic_text, finalize=True):
"""
Run a tahoe process from its node_dir.
@ -185,10 +205,8 @@ def _run_node(reactor, node_dir, request, magic_text):
magic_text = "client running"
protocol = _MagicTextProtocol(magic_text)
# on windows, "tahoe start" means: run forever in the foreground,
# but on linux it means daemonize. "tahoe run" is consistent
# between platforms.
# "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
# "start" command.
transport = _tahoe_runner_optional_coverage(
protocol,
reactor,
@ -201,7 +219,8 @@ def _run_node(reactor, node_dir, request, magic_text):
)
transport.exited = protocol.exited
request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
if finalize:
request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited))
# XXX abusing the Deferred; should use .when_magic_seen() pattern
@ -220,7 +239,8 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
magic_text=None,
needed=2,
happy=3,
total=4):
total=4,
finalize=True):
"""
Helper to create a single node, run it and return the instance
spawnProcess returned (ITransport)
@ -256,14 +276,19 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
def created(_):
config_path = join(node_dir, 'tahoe.cfg')
config = get_config(config_path)
set_config(config, 'node', 'log_gatherer.furl', flog_gatherer)
write_config(config_path, config)
set_config(
config,
u'node',
u'log_gatherer.furl',
flog_gatherer.decode("utf-8"),
)
write_config(FilePath(config_path), config)
created_d.addCallback(created)
d = Deferred()
d.callback(None)
d.addCallback(lambda _: created_d)
d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text))
d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text, finalize=finalize))
return d
@ -383,17 +408,13 @@ def await_file_vanishes(path, timeout=10):
raise FileShouldVanishException(path, timeout)
def cli(request, reactor, node_dir, *argv):
def cli(node, *argv):
"""
Run a tahoe CLI subcommand for a given node, optionally running
under coverage if '--coverage' was supplied.
Run a tahoe CLI subcommand for a given node in a blocking manner, returning
the output.
"""
proto = _CollectOutputProtocol()
_tahoe_runner_optional_coverage(
proto, reactor, request,
['--node-directory', node_dir] + list(argv),
)
return proto.done
arguments = ["tahoe", '--node-directory', node.node_dir]
return check_output(arguments + list(argv))
def node_url(node_dir, uri_fragment):
@ -498,3 +519,36 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2):
tahoe,
)
)
def generate_ssh_key(path):
"""Create a new SSH private/public key pair."""
key = RSAKey.generate(2048)
key.write_private_key_file(path)
with open(path + ".pub", "wb") as f:
f.write(b"%s %s" % (key.get_name(), key.get_base64()))
def run_in_thread(f):
"""Decorator for integration tests that runs code in a thread.
Because we're using pytest_twisted, tests that rely on the reactor are
expected to return a Deferred and use async APIs so the reactor can run.
In the case of the integration test suite, it launches nodes in the
background using Twisted APIs. The nodes stdout and stderr is read via
Twisted code. If the reactor doesn't run, reads don't happen, and
eventually the buffers fill up, and the nodes block when they try to flush
logs.
We can switch to Twisted APIs (treq instead of requests etc.), but
sometimes it's easier or expedient to just have a blocking test. So this
decorator allows you to run the test in a thread, and the reactor can keep
running in the main thread.
See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
"""
@wraps(f)
def test(*args, **kwargs):
return deferToThread(lambda: f(*args, **kwargs))
return test

8
misc/build_helpers/run-deprecations.py
View File

@ -20,7 +20,7 @@ class Options(usage.Options):
self["args"] = list(args)
description = """Run as:
PYTHONWARNINGS=default::DeprecationWarning python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] COMMAND ARGS..
python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] COMMAND ARGS..
"""
class RunPP(protocol.ProcessProtocol):
@ -79,12 +79,6 @@ def run_command(main):
(command, os.environ.get("PATH")))
exe = executables[0]
pw = os.environ.get("PYTHONWARNINGS")
DDW = "default::DeprecationWarning"
if pw != DDW:
print("note: $PYTHONWARNINGS is '%s', not the expected %s" % (pw, DDW))
sys.stdout.flush()
pp = RunPP()
pp.d = defer.Deferred()
pp.stdout = io.BytesIO()

1
misc/build_helpers/show-tool-versions.py
View File

@ -143,7 +143,6 @@ print_py_pkg_ver('coverage')
print_py_pkg_ver('cryptography')
print_py_pkg_ver('foolscap')
print_py_pkg_ver('mock')
print_py_pkg_ver('Nevow', 'nevow')
print_py_pkg_ver('pyasn1')
print_py_pkg_ver('pycparser')
print_py_pkg_ver('cryptography')

8
misc/coding_tools/check-debugging.py
View File

@ -11,8 +11,12 @@ umids = {}
for starting_point in sys.argv[1:]:
for root, dirs, files in os.walk(starting_point):
for fn in [f for f in files if f.endswith(".py")]:
fn = os.path.join(root, fn)
for f in files:
if not f.endswith(".py"):
continue
if f == "check-debugging.py":
continue
fn = os.path.join(root, f)
for lineno,line in enumerate(open(fn, "r").readlines()):
lineno = lineno+1
mo = re.search(r"\.setDebugging\(True\)", line)

3
mypy.ini Normal file
View File

@ -0,0 +1,3 @@
[mypy]
ignore_missing_imports = True
plugins=mypy_zope:plugin

1
newsfragments/1549.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS now requires Twisted 19.10.0 or newer. As a result, it now has a transitive dependency on bcrypt.

0
newsfragments/2920.minor Normal file
View File

0
newsfragments/2928.minor Normal file
View File

0
newsfragments/3283.minor Normal file
View File

0
newsfragments/3314.minor Normal file
View File

0
newsfragments/3384.minor Normal file
View File

1
newsfragments/3399.feature Normal file
View File

@ -0,0 +1 @@
Added 'typechecks' environment for tox running mypy and performing static typechecks.

0
newsfragments/3428.minor Normal file
View File

0
newsfragments/3432.minor Normal file
View File

1
newsfragments/3433.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS no longer depends on Nevow.

0
newsfragments/3434.minor Normal file
View File

0
newsfragments/3435.minor Normal file
View File

0
newsfragments/3454.minor Normal file
View File

0
newsfragments/3459.minor Normal file
View File

0
newsfragments/3460.minor Normal file
View File

0
newsfragments/3465.minor Normal file
View File

0
newsfragments/3466.minor Normal file
View File

0
newsfragments/3467.minor Normal file
View File

0
newsfragments/3468.minor Normal file
View File

0
newsfragments/3470.minor Normal file
View File

0
newsfragments/3471.minor Normal file
View File

0
newsfragments/3472.minor Normal file
View File

0
newsfragments/3473.minor Normal file
View File

0
newsfragments/3474.minor Normal file
View File

0
newsfragments/3475.minor Normal file
View File

0
newsfragments/3477.minor Normal file
View File

1
newsfragments/3478.minor Normal file
View File

@ -0,0 +1 @@

0
newsfragments/3479.minor Normal file
View File

0
newsfragments/3481.minor Normal file
View File

0
newsfragments/3482.minor Normal file
View File

0
newsfragments/3483.minor Normal file
View File

0
newsfragments/3485.minor Normal file
View File

1
newsfragments/3486.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS now requires the `netifaces` Python package and no longer requires the external `ip`, `ifconfig`, or `route.exe` executables.

0
newsfragments/3488.minor Normal file
View File

0
newsfragments/3490.minor Normal file
View File

0
newsfragments/3491.minor Normal file
View File

0
newsfragments/3492.minor Normal file
View File

0
newsfragments/3493.minor Normal file
View File

0
newsfragments/3496.minor Normal file
View File

1
newsfragments/3497.installation Normal file
View File

@ -0,0 +1 @@
The Tahoe-LAFS project no longer commits to maintaining binary packages for all dependencies at <https://tahoe-lafs.org/deps>. Please use PyPI instead.

0
newsfragments/3499.minor Normal file
View File

0
newsfragments/3500.minor Normal file
View File

0
newsfragments/3501.minor Normal file
View File

0
newsfragments/3502.minor Normal file
View File

1
newsfragments/3503.other Normal file
View File

@ -0,0 +1 @@
The specification section of the Tahoe-LAFS documentation now includes explicit discussion of the security properties of Foolscap "fURLs" on which it depends.

1
newsfragments/3504.configuration Normal file
View File

@ -0,0 +1 @@
The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file.

1
newsfragments/3509.bugfix Normal file
View File

@ -0,0 +1 @@
Fix regression that broke flogtool results on Python 2.

1
newsfragments/3510.bugfix Normal file
View File

@ -0,0 +1 @@
Fix a logging regression on Python 2 involving unicode strings.

0
newsfragments/3511.minor Normal file
View File

0
newsfragments/3513.minor Normal file
View File

0
newsfragments/3514.minor Normal file
View File

Some files were not shown because too many files have changed in this diff Show More