ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-08 11:24:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #504 from exarkun/2929.circleci

Browse Source 
Configure CircleCI for much of the Linux testing
This commit is contained in:
Jean-Paul Calderone 2018-07-03 12:32:51 -04:00 committed by GitHub
commit 814203961d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 357 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.circleci/bootstrap-test-environment.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash -e
PROJECT=$1
shift
EXTRA_PACKAGES=$1
shift
# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try to test this
# checkout.
mv "${PROJECT}" /tmp/project
# Python build/install toolchain wants to write to the source checkout, too.
chown --recursive nobody:nogroup /tmp/project
apt-get --quiet --yes install \
sudo \
build-essential \
python2.7 \
python2.7-dev \
libffi-dev \
libssl-dev \
libyaml-dev \
${EXTRA_PACKAGES}

261
.circleci/config.yml Normal file
View File

@ -0,0 +1,261 @@
# https://circleci.com/docs/2.0/
version: 2
workflows:
version: 2
ci:
jobs:
- "lint"
- "debian-8"
- "debian-9"
- "ubuntu-16.04"
- "ubuntu-18.04"
- "centos-7"
- "fedora-27"
- "fedora-28"
- "magic-folder-ubuntu-14.04"
- "deprecations"
- "c-locale"
jobs:
lint:
docker:
- image: "circleci/python:2"
steps:
- "checkout"
- run:
name: "Install tox"
command: |
pip install --user tox
- run:
name: "Static-ish code checks"
command: |
~/.local/bin/tox -e codechecks
debian-8: &DEBIAN
docker:
- image: "debian:8"
environment: &UTF_8_ENVIRONMENT
# Tell Hypothesis which configuration we want it to use.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
# Tell the C runtime things about character encoding (mainly to do with
# filenames and argv).
LANG: "en_US.UTF-8"
# The package name for this software varies somewhat across distros.
EXTRA_PACKAGES: "virtualenv"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
steps:
- run: &INSTALL_GIT
node: "Install Git"
command: |
apt-get --quiet update
apt-get --quiet --yes install git
- "checkout"
- run: &BOOTSTRAP_TEST_ENVIRONMENT
name: "Bootstrap test environment"
command: |
~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
# pip cannot install packages if the working directory is not
# readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
- run: &RUN_TESTS
name: "Run test suite"
# Something about when it re-uses an existing environment blows up
# if the working directory is not readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
# in the project source checkout.
path: "/tmp/project/_trial_temp/test.log"
- store_artifacts: &STORE_OTHER_ARTIFACTS
# Store any other artifacts, too. This is handy to allow other jobs
# sharing most of the definition of this one to be able to
# contribute artifacts easily.
path: "/tmp/artifacts"
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
working_directory: "/tmp/project"
command: |
/tmp/tests/bin/codecov
debian-9:
<<: *DEBIAN
docker:
- image: "debian:9"
c-locale:
<<: *DEBIAN
docker:
- image: "debian:9"
environment:
<<: *UTF_8_ENVIRONMENT
LANG: "C"
deprecations:
<<: *DEBIAN
docker:
- image: "debian:9"
environment:
<<: *UTF_8_ENVIRONMENT
# Select the deprecations tox environments.
TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
# Put the logs somewhere we can report them.
TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
ubuntu-16.04:
<<: *DEBIAN
docker:
- image: "ubuntu:16.04"
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for en_US LANG setting.
EXTRA_PACKAGES: "virtualenv language-pack-en"
ubuntu-18.04:
<<: *DEBIAN
docker:
- image: "ubuntu:18.04"
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for automatic address detection/assignment.
EXTRA_PACKAGES: "virtualenv iproute2"
centos-7: &RHEL_DERIV
docker:
- image: "centos:7"
environment: *UTF_8_ENVIRONMENT
steps:
- run:
node: "Install Git"
command: |
yum install --assumeyes git
- "checkout"
- run:
name: "Bootstrap test environment"
working_directory: "/tmp"
command: |
# Avoid the /nonexistent home directory in nobody's /etc/passwd
# entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try
# to test this checkout.
mv /root/project /tmp/project
# Python build/install toolchain wants to write to the source
# checkout, too.
chown --recursive nobody:nobody /tmp/project
yum install --assumeyes \
sudo \
make automake gcc gcc-c++ \
python \
python-devel \
libffi-devel \
openssl-devel \
libyaml-devel \
python-virtualenv
# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
yum install --assumeyes \
net-tools
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
fedora-27:
<<: *RHEL_DERIV
docker:
- image: "fedora:27"
fedora-28:
<<: *RHEL_DERIV
docker:
- image: "fedora"
magic-folder-ubuntu-14.04:
machine:
enabled: true
image: "circleci/classic:201711-01"
environment:
<<: *UTF_8_ENVIRONMENT
EXTRA_PACKAGES: "python-virtualenv"
TAHOE_LAFS_TOX_ARGS: "-- allmydata.test.test_magic_folder"
# Unfortunately, duplicate all the steps here but run with `sudo`.
steps:
- run:
node: "Install Git"
command: |
sudo apt-get --quiet update
sudo apt-get --quiet --yes install git
- "checkout"
- run:
<<: *BOOTSTRAP_TEST_ENVIRONMENT
command: |
sudo ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
- run:
<<: *SETUP_VIRTUALENV
command: |
env
sudo /tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
- run:
<<: *RUN_TESTS
command: |
env
sudo /tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE

10
.circleci/run-build-locally.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
CIRCLE_TOKEN=efb53124be82dd4b3153bc0e3f60de71da629d59
curl --user ${CIRCLE_TOKEN}: \
--request POST \
--form revision=$(git rev-parse HEAD) \
--form config=@config.yml \
--form notify=false \
https://circleci.com/api/v1.1/project/github/exarkun/tahoe-lafs/tree/2929.circleci

16
.circleci/run-tests.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash -e
TAHOE_LAFS_TOX_ENVIRONMENT=$1
shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable
# files being unreadable).
#
# Also run with /tmp as a workdir because the non-root user won't be able to
# create the tox working filesystem state in the source checkout because it is
# owned by root.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}

14
.circleci/setup-virtualenv.sh Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash -e
TAHOE_LAFS_TOX_ENVIRONMENT=$1
shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Set up the virtualenv as a non-root user so we can run the test suite as a
# non-root user. See below.
sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov
# Get everything installed in it, too.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}

5
README.rst
View File

@ -10,7 +10,7 @@ function correctly, preserving your privacy and security.
For full documentation, please see
http://tahoe-lafs.readthedocs.io/en/latest/ .
|readthedocs| |travis| |codecov|
|readthedocs| |travis| |circleci| |codecov|
INSTALLING
==========
@ -99,6 +99,9 @@ slides.
:alt: build status
:target: https://travis-ci.org/tahoe-lafs/tahoe-lafs
.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
:target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
.. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
:alt: test coverage percentage
:target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master

24
src/allmydata/test/test_util.py
View File

@ -643,9 +643,31 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
self.failIfEqual(new_mode, 0766)
def test_create_long_path(self):
"""
Even for paths with total length greater than 260 bytes,
``fileutil.abspath_expanduser_unicode`` produces a path on which other
path-related APIs can operate.
https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
documents certain Windows-specific path length limitations this test
is specifically intended to demonstrate can be overcome.
"""
workdir = u"test_create_long_path"
fileutil.make_dirs(workdir)
long_path = fileutil.abspath_expanduser_unicode(os.path.join(workdir, u'x'*255))
base_path = fileutil.abspath_expanduser_unicode(workdir)
base_length = len(base_path)
# Construct a path /just/ long enough to exercise the important case.
# It would be nice if we could just use a seemingly globally valid
# long file name (the `x...` portion) here - for example, a name 255
# bytes long- and a previous version of this test did just that.
# However, aufs imposes a 242 byte length limit on file names. Most
# other POSIX filesystems do allow names up to 255 bytes. It's not
# clear there's anything we can *do* about lower limits, though, and
# POSIX.1-2017 (and earlier) only requires that the maximum be at
# least 14 (!!!) bytes.
long_path = os.path.join(base_path, u'x' * (261 - base_length))
def _cleanup():
fileutil.remove(long_path)
self.addCleanup(_cleanup)

9
tox.ini
View File

@ -25,7 +25,6 @@ deps = incremental
usedevelop = True
extras = test
commands =
pyflakes src static misc setup.py
tahoe --version
trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
@ -37,7 +36,6 @@ commands =
[testenv:coverage]
# coverage (with --branch) takes about 65% longer to run
commands =
pyflakes src static misc setup.py
tahoe --version
coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
coverage xml
@ -49,15 +47,12 @@ commands =
python misc/coding_tools/check-debugging.py
python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
python misc/coding_tools/check-miscaptures.py
# note: check-interfaces.py imports everything, so it must be run
# from a populated virtualenv
python misc/coding_tools/check-interfaces.py
[testenv:deprecations]
setenv =
PYTHONWARNINGS=default::DeprecationWarning
commands =
python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
[testenv:upcoming-deprecations]
setenv =
@ -67,7 +62,7 @@ deps =
git+https://github.com/warner/foolscap
commands =
flogtool --version
python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
[testenv:checkmemory]
commands =