ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-21 10:01:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 2932.encourage-zcash-workflow

Browse Source
This commit is contained in:
Jean-Paul Calderone 2018-07-17 13:15:29 -04:00
commit 0445dd79cd
5 changed files with 193 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
.circleci/config.yml
View File

@ -64,10 +64,14 @@ jobs:
TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
# Convince all of our pip invocations to look at the cached wheelhouse
# we maintain.
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
steps:
- run: &INSTALL_GIT
node: "Install Git"
name: "Install Git"
command: |
apt-get --quiet update
apt-get --quiet --yes install git
@ -79,14 +83,60 @@ jobs:
command: |
~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
- restore_cache: &RESTORE_HTTP_CACHE
name: "Restoring pip HTTP cache"
keys:
# An exact match on the http cache key is great. It should have
# exactly the packages (tgz, whl, whatever) we need.
- v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A prefix match is okay too. It might have a
# partially-overlapping set of packages. That's a head-start, at
# least. We might have to download a few more things but at least
# we saved a little time. After we download some more stuff we'll
# create a new cache entry with the full key above and the next
# build will get a better cache hit.
- v5-pip-http-
- restore_cache: &RESTORE_WHEELHOUSE
name: "Restoring wheelhouse"
keys:
# As above, an exact match is great. Here, we also need to
# include the job name to make sure the platform ABI matches.
# There are binary wheels in this wheelhouse and we're not taking
# care to make manylinux1 wheels. The binary wheels in this cache
# will only work on some Linux distros.
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A partial match is okay too. It'll get us at least some of the
# wheels. We do need to keep the job name as part of the key or
# we might get binary wheels build against an incompatible ABI and
# we won't be able to use them (and they'll break the build rather
# than being ignored).
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
# pip cannot install packages if the working directory is not
# readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
/tmp/project/.circleci/setup-virtualenv.sh \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
- save_cache: &SAVE_HTTP_CACHE
name: "Saving pip HTTP cache"
key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
# Perfectly valid for Linux. Note we exclude the wheel cache
# because we want this cache to be valid across all platforms and
# the wheels in the pip wheel cache are not necessarily so.
- "/tmp/nobody/.cache/pip/http"
- save_cache: &SAVE_WHEELHOUSE
name: "Caching wheelhouse"
key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
- *WHEELHOUSE_PATH
- run: &RUN_TESTS
name: "Run test suite"
@ -94,8 +144,16 @@ jobs:
# if the working directory is not readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
/tmp/project/.circleci/run-tests.sh \
/tmp/artifacts \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
# trial output gets directed straight to a log. avoid the circleci
# timeout while the test suite runs.
no_output_timeout: "20m"
- store_test_results: &STORE_TEST_RESULTS
path: "/tmp/artifacts/junit"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
@ -174,7 +232,7 @@ jobs:
steps:
- run:
node: "Install Git"
name: "Install Git"
command: |
yum install --assumeyes git
@ -210,9 +268,17 @@ jobs:
yum install --assumeyes \
net-tools
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
@ -238,7 +304,7 @@ jobs:
steps:
- run:
node: "Install Git"
name: "Install Git"
command: |
slackpkg update
slackpkg install openssh-7.4p1 git-2.14.4 </dev/null
@ -262,6 +328,7 @@ jobs:
chown --recursive nobody:nobody /tmp/project
slackpkg install \
ca-certificates \
sudo-1.8.20p2 \
make-4.1 \
automake-1.15 \
@ -290,9 +357,17 @@ jobs:
python get-pip.py
pip install virtualenv
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE

31
.circleci/run-tests.sh
View File

@ -1,11 +1,23 @@
#!/bin/bash -e
ARTIFACTS=$1
shift
TAHOE_LAFS_TOX_ENVIRONMENT=$1
shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Make sure we can actually write things to this directory.
sudo --user nobody mkdir -p "${ARTIFACTS}"
SUBUNIT2="${ARTIFACTS}"/results.subunit2
# Use an intermediate directory here because CircleCI extracts some label
# information from its name.
JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable
# files being unreadable).
@ -13,4 +25,21 @@ shift || :
# Also run with /tmp as a workdir because the non-root user won't be able to
# create the tox working filesystem state in the source checkout because it is
# owned by root.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
#
# Send the output directly to a file because transporting the binary subunit2
# via tox and then scraping it out is hideous and failure prone.
sudo \
SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file" \
PIP_NO_INDEX="1" \
--set-home \
--user nobody \
/tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS}
# Create a junitxml results area.
mkdir -p "$(dirname "${JUNITXML}")"
/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"

55
.circleci/setup-virtualenv.sh
View File

@ -6,15 +6,58 @@ shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
REPORTING_DEPS="python-subunit junitxml subunitreporter"
# Make sure the ownership of the pip cache directory is correct. The CircleCI
# cache management operations seem to mess it up. The cache directory might
# not exist if there was no matching cache to restore.
[ -e /tmp/nobody/.cache ] && chown --recursive nobody /tmp/nobody/.cache
# Set up the virtualenv as a non-root user so we can run the test suite as a
# non-root user. See below.
sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
# Slackware has non-working SSL support in setuptools until certifi is
# installed. SSL support in setuptools is needed in case packages use
# `setup_requires` which gets satisfied by setuptools instead of by pip.
# txi2p (vcversioner) is one such package. Twisted (incremental) is another.
sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem. Make sure this step comes before any other attempts to
# install things using pip!
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install certifi
# Get a new, awesome version of pip and setuptools. For example, the
# distro-packaged virtualenv's pip may not know about wheels.
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install --upgrade pip setuptools wheel
# Populate the wheelhouse, if necessary.
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip \
wheel \
--wheel-dir "${WHEELHOUSE_PATH}" \
/tmp/project ${TEST_DEPS} ${REPORTING_DEPS}
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
# Get everything else installed in it, too.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
--notest \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS}

32
NEWS.rst
View File

@ -4,7 +4,7 @@
User-Visible Changes in Tahoe-LAFS
==================================
Release 1.13.0 (18-May-2018)
Release 1.13.0 (30-July-2018)
''''''''''''''''''''''''''''
New Features
@ -49,7 +49,10 @@ Bug Fixes in Core
Some bugs with pidfile handling were fixed (`PR440`_ and `PR450`_)
meaning invalid pidfiles are now deleted. Error-messages related to
``tahoe.cfg`` now include the full path to the file.
``tahoe.cfg`` now include the full path to the file. `PR501`_ fixes
"address already in use" test failures. `PR502`_ fixes ticket #2926
("tahoe status" failures). `PR487`_ fixes ticket #1455 (setting
``X-Frame-Options: DENY``)
Web UI Changes
@ -65,19 +68,20 @@ Magic Folder Changes
Multiple magic-folders in a single Tahoe client are now
supported. Bugs with ``.backup`` files have been fixed, meaning
spurious ``.backup`` files will be produced less often
(`PR448`_). Handling of default umask on new magic-folder files is
spurious ``.backup`` files will be produced less often (`PR448`_,
`PR475`_). Handling of default umask on new magic-folder files is
fixed in `PR458`_. The user mtime value is now correctly preserved
(`PR457`_).
A bug in ``tahoe magic-folder status`` causing active operations to
sometimes not show up is fixed (`PR461`_).
sometimes not show up is fixed (`PR461`_). If a directory is missing,
it is created (`PR492`_).
Raw Pull Requests
-----------------
In total, 44 Pull Requests were merged for this release, including
In total, 50 Pull Requests were merged for this release, including
contributions of code or review from 15 different GitHub users. Thanks
everyone! A complete list of these PRs and contributions:
@ -89,6 +93,7 @@ everyone! A complete list of these PRs and contributions:
`PR407`_: `david415`_ (with `meejah`_, `warner`_)
`PR409`_: `str4d`_ (with `warner`_)
`PR410`_: `tpltnt`_ (with `warner`_)
`PR411`_: `tpltnt`_ (with `warner`_, `meejah`_)
`PR412`_: `tpltnt`_ (with `warner`_)
`PR414`_: `tpltnt`_ (with `meejah`_, `warner`_)
`PR416`_: `david415`_, `meejah`_, `markberger`_, `warner`_
@ -115,6 +120,7 @@ everyone! A complete list of these PRs and contributions:
`PR452`_: `meejah`_ (with `tpltnt`_)
`PR453`_: `meejah`_
`PR454`_: `meejah`_ (with `tpltnt`_, `meejah`_, `warner`_)
`PR455`_: `tpltnt`_ (with `meejah`_)
`PR456`_: `meejah`_ (with `meejah`_)
`PR457`_: `meejah`_ (with `crwood`_, `tpltnt`_)
`PR458`_: `meejah`_ (with `tpltnt`_)
@ -124,7 +130,20 @@ everyone! A complete list of these PRs and contributions:
`PR470`_: `meejah`_ (with `exarkun`_, `tpltnt`_, `warner`_)
`PR472`_: `exarkun`_, `meskio`_
`PR474`_: `exarkun`_
`PR475`_: `meejah`_ (with `exarkun`_)
`PR482`_: `crwood`_ (with `warner`_)
`PR485`_: `warner`_
`PR486`_: `exarkun`_ (with `warner`_)
`PR487`_: `exarkun`_ (with `tpltnt`_)
`PR489`_: `exarkun`_
`PR490`_: `exarkun`_
`PR491`_: `exarkun`_ (with `meejah`_)
`PR492`_: `exarkun`_ (with `meejah`_, `tpltnt`_)
`PR493`_: `exarkun`_ (with `meejah`_)
`PR494`_: `exarkun`_ (with `meejah`_)
`PR497`_: `meejah`_ (with `multikatt`_, `exarkun`_)
`PR499`_: `exarkun`_ (with `meejah`_)
`PR501`_: `exarkun`_ (with `meejah`_)
`PR502`_: `exarkun`_ (with `meejah`_)
@ -228,6 +247,7 @@ improvements which shouldn't have any user-visible effects:
.. _markberger: https://github.com/markberger
.. _meejah: https://github.com/meejah
.. _meskio: https://github.com/meskio
.. _multikatt: https://github.com/multikatt
.. _pataquets: https://github.com/pataquets
.. _str4d: https://github.com/str4d
.. _tpltnt: https://github.com/tpltnt

8
tox.ini
View File

@ -13,14 +13,16 @@ skipsdist = True
[testenv]
basepython=python2.7
passenv = TAHOE_LAFS_* USERPROFILE HOMEDRIVE HOMEPATH
passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem.
deps = certifi
deps =
certifi
subunitreporter
# We add usedevelop=True for speed, and extras=test to get things like "mock"
# that are required for our unit tests.
usedevelop = True
@ -59,6 +61,8 @@ commands =
setenv =
PYTHONWARNINGS=default::DeprecationWarning
deps =
# Take the base deps as well!
{[testenv]deps}
git+https://github.com/twisted/twisted
git+https://github.com/warner/foolscap
commands =