tahoe-lafs/.circleci/config.yml

483 lines
15 KiB
YAML
Raw Normal View History

# https://circleci.com/docs/2.0/
version: 2
workflows:
version: 2
ci:
jobs:
# Platforms
2018-06-16 20:06:14 +00:00
- "debian-9"
- "debian-8":
requires:
- "debian-9"
2018-06-16 20:06:14 +00:00
- "ubuntu-18.04"
- "ubuntu-16.04":
requires:
- "ubuntu-18.04"
- "fedora-29"
- "fedora-28":
requires:
- "fedora-29"
- "centos-7"
2018-07-03 18:52:10 +00:00
- "slackware-14.2"
# Other assorted tasks and configurations
- "lint"
2018-06-16 20:06:14 +00:00
- "deprecations"
2018-06-19 12:37:18 +00:00
- "c-locale"
2019-03-20 19:56:01 +00:00
# Any locale other than C or UTF-8.
- "another-locale"
- "integration":
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
- "debian-9"
2018-06-15 12:57:50 +00:00
jobs:
lint:
docker:
- image: "circleci/python:2"
steps:
- "checkout"
- run:
name: "Install tox"
command: |
2018-06-08 18:47:00 +00:00
pip install --user tox
- run:
name: "Static-ish code checks"
command: |
2018-06-08 18:47:00 +00:00
~/.local/bin/tox -e codechecks
2018-06-08 18:58:25 +00:00
debian-9: &DEBIAN
2018-06-08 18:58:25 +00:00
docker:
- image: "debian:9"
2018-06-08 18:58:25 +00:00
environment: &UTF_8_ENVIRONMENT
# Tell Hypothesis which configuration we want it to use.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
# Tell the C runtime things about character encoding (mainly to do with
2018-06-14 16:32:49 +00:00
# filenames and argv).
LANG: "en_US.UTF-8"
2018-06-15 19:34:21 +00:00
# The package name for this software varies somewhat across distros.
EXTRA_PACKAGES: "virtualenv"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
2018-06-15 17:34:17 +00:00
# Additional arguments to pass to tox.
2018-07-09 19:55:32 +00:00
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
# Convince all of our pip invocations to look at the cached wheelhouse
# we maintain.
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
2018-06-13 16:36:51 +00:00
2018-06-08 18:58:25 +00:00
steps:
- run: &INSTALL_GIT
2018-07-09 17:31:56 +00:00
name: "Install Git"
command: |
2018-06-15 18:54:10 +00:00
apt-get --quiet update
apt-get --quiet --yes install git
2018-06-08 18:58:25 +00:00
- "checkout"
- run: &BOOTSTRAP_TEST_ENVIRONMENT
name: "Bootstrap test environment"
command: |
~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
- restore_cache: &RESTORE_HTTP_CACHE
2018-07-10 18:03:57 +00:00
name: "Restoring pip HTTP cache"
keys:
# An exact match on the http cache key is great. It should have
# exactly the packages (tgz, whl, whatever) we need.
2018-07-10 17:24:55 +00:00
- v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A prefix match is okay too. It might have a
# partially-overlapping set of packages. That's a head-start, at
# least. We might have to download a few more things but at least
# we saved a little time. After we download some more stuff we'll
# create a new cache entry with the full key above and the next
# build will get a better cache hit.
2018-07-10 17:24:55 +00:00
- v5-pip-http-
- restore_cache: &RESTORE_WHEELHOUSE
2018-07-10 18:03:57 +00:00
name: "Restoring wheelhouse"
keys:
# As above, an exact match is great. Here, we also need to
# include the job name to make sure the platform ABI matches.
# There are binary wheels in this wheelhouse and we're not taking
# care to make manylinux1 wheels. The binary wheels in this cache
# will only work on some Linux distros.
2018-07-10 17:24:55 +00:00
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A partial match is okay too. It'll get us at least some of the
# wheels. We do need to keep the job name as part of the key or
# we might get binary wheels build against an incompatible ABI and
# we won't be able to use them (and they'll break the build rather
# than being ignored).
2018-07-10 17:24:55 +00:00
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
# pip cannot install packages if the working directory is not
# readable.
working_directory: "/tmp"
2018-06-15 19:14:55 +00:00
command: |
2018-07-06 15:32:12 +00:00
/tmp/project/.circleci/setup-virtualenv.sh \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
2018-06-08 18:58:25 +00:00
- save_cache: &SAVE_HTTP_CACHE
2018-07-10 18:03:57 +00:00
name: "Saving pip HTTP cache"
2018-07-10 17:24:55 +00:00
key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
# Perfectly valid for Linux. Note we exclude the wheel cache
# because we want this cache to be valid across all platforms and
# the wheels in the pip wheel cache are not necessarily so.
- "/tmp/nobody/.cache/pip/http"
- save_cache: &SAVE_WHEELHOUSE
2018-07-10 18:03:57 +00:00
name: "Caching wheelhouse"
2018-07-10 17:24:55 +00:00
key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
- *WHEELHOUSE_PATH
- run: &RUN_TESTS
2018-06-08 18:58:25 +00:00
name: "Run test suite"
# Something about when it re-uses an existing environment blows up
# if the working directory is not readable.
working_directory: "/tmp"
2018-06-15 19:14:55 +00:00
command: |
/tmp/project/.circleci/run-tests.sh \
"${ARTIFACTS_OUTPUT_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
2018-07-08 23:06:34 +00:00
# trial output gets directed straight to a log. avoid the circleci
# timeout while the test suite runs.
no_output_timeout: "20m"
- store_test_results: &STORE_TEST_RESULTS
path: "/tmp/artifacts/junit"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
# in the project source checkout.
path: "/tmp/project/_trial_temp/test.log"
- store_artifacts: &STORE_OTHER_ARTIFACTS
# Store any other artifacts, too. This is handy to allow other jobs
# sharing most of the definition of this one to be able to
# contribute artifacts easily.
path: "/tmp/artifacts"
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
2018-06-16 14:19:45 +00:00
working_directory: "/tmp/project"
command: |
/tmp/tests/bin/codecov
2018-06-14 15:43:59 +00:00
debian-8:
<<: *DEBIAN
2018-06-08 19:52:29 +00:00
docker:
- image: "debian:8"
2018-06-08 19:52:29 +00:00
2018-06-13 17:59:52 +00:00
2018-06-19 12:37:18 +00:00
c-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
LANG: "C"
2019-03-20 19:56:01 +00:00
another-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
# aka "Latin 1"
LANG: "en_US.ISO-8859-1"
2018-06-15 12:57:50 +00:00
deprecations:
<<: *DEBIAN
2018-06-15 12:57:50 +00:00
environment:
<<: *UTF_8_ENVIRONMENT
# Select the deprecations tox environments.
2018-06-15 12:57:50 +00:00
TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
# Put the logs somewhere we can report them.
TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
2018-06-15 12:57:50 +00:00
integration:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
# Select the integration tests tox environments.
TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
steps:
# DRY, YAML-style. See the debian-9 steps.
- run: *INSTALL_GIT
- "checkout"
- run: *BOOTSTRAP_TEST_ENVIRONMENT
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
- run:
name: "Install Tor"
command: |
# Don't forget that we moved the source tree to /tmp.
/tmp/project/integration/install-tor.sh
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
2018-06-14 15:44:39 +00:00
ubuntu-16.04:
<<: *DEBIAN
docker:
- image: "ubuntu:16.04"
2018-06-14 16:32:49 +00:00
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for en_US LANG setting.
2018-06-15 19:34:21 +00:00
EXTRA_PACKAGES: "virtualenv language-pack-en"
2018-06-14 16:32:49 +00:00
2018-06-14 15:44:39 +00:00
2018-06-14 14:46:45 +00:00
ubuntu-18.04:
<<: *DEBIAN
docker:
- image: "ubuntu:18.04"
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for automatic address detection/assignment.
2018-06-15 19:34:21 +00:00
EXTRA_PACKAGES: "virtualenv iproute2"
2018-06-14 14:46:45 +00:00
2018-06-14 14:09:42 +00:00
centos-7: &RHEL_DERIV
2018-06-13 17:59:52 +00:00
docker:
- image: "centos:7"
2018-06-14 16:32:49 +00:00
environment: *UTF_8_ENVIRONMENT
2018-06-13 17:59:52 +00:00
steps:
- run:
2018-07-09 17:31:56 +00:00
name: "Install Git"
2018-06-13 17:59:52 +00:00
command: |
yum install --assumeyes git
- "checkout"
- run:
name: "Bootstrap test environment"
working_directory: "/tmp"
command: |
# Avoid the /nonexistent home directory in nobody's /etc/passwd
# entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try
# to test this checkout.
mv /root/project /tmp/project
# Python build/install toolchain wants to write to the source
# checkout, too.
chown --recursive nobody:nobody /tmp/project
yum install --assumeyes \
sudo \
make automake gcc gcc-c++ \
python \
python-devel \
libffi-devel \
openssl-devel \
libyaml-devel
yum install --assumeyes /usr/bin/virtualenv
2018-06-13 17:59:52 +00:00
# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
yum install --assumeyes \
net-tools
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
2018-07-09 19:51:10 +00:00
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
2018-06-14 14:09:42 +00:00
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
fedora-28:
<<: *RHEL_DERIV
2018-06-14 14:09:42 +00:00
docker:
- image: "fedora:28"
2018-06-14 14:09:42 +00:00
2018-06-14 14:11:49 +00:00
fedora-29:
<<: *RHEL_DERIV
2018-06-14 14:11:49 +00:00
docker:
- image: "fedora:29"
2018-06-15 17:34:17 +00:00
2018-07-03 18:52:10 +00:00
slackware-14.2:
docker:
- image: "vbatts/slackware:14.2"
environment: *UTF_8_ENVIRONMENT
steps:
- run:
2018-07-09 17:31:56 +00:00
name: "Install Git"
2018-07-03 18:52:10 +00:00
command: |
slackpkg update
# Be careful with slackpkg. If the package name given doesn't
# match anything, slackpkg still claims to succeed but you're
# totally screwed. Slackware updates versions of packaged
# software so including too much version prefix is a good way to
# have your install commands suddenly begin not installing
# anything.
slackpkg install openssh-7 git-2 </dev/null
2018-07-03 18:52:10 +00:00
- "checkout"
- run:
name: "Bootstrap test environment"
working_directory: "/tmp"
command: |
# Avoid the /nonexistent home directory in nobody's /etc/passwd
# entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try
# to test this checkout.
mv /root/project /tmp/project
# Python build/install toolchain wants to write to the source
# checkout, too.
chown --recursive nobody:nobody /tmp/project
slackpkg install \
ca-certificates \
sudo-1 \
make-4 \
automake-1 \
2018-07-04 20:01:54 +00:00
kernel-headers \
glibc-2 \
binutils-2 \
gcc-5 \
gcc-g++-5 \
python-2 \
libffi-3 \
libyaml-0 \
sqlite-3 \
icu4c-56 \
libmpc-1 </dev/null
slackpkg upgrade \
openssl-1 </dev/null
2018-07-03 19:40:48 +00:00
# neither virtualenv nor pip is packaged.
# do it the hard way.
# and it is extra hard since it is slackware.
slackpkg install \
cyrus-sasl-2 \
curl-7 </dev/null
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py
pip install virtualenv
2018-07-03 18:52:10 +00:00
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
2018-07-03 18:52:10 +00:00
- run: *SETUP_VIRTUALENV
2018-07-09 19:51:10 +00:00
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
2018-07-03 18:52:10 +00:00
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
2018-07-03 18:52:10 +00:00
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
build-images:
# This job builds Docker images that have as much of the setup as we can
# manage already done and baked in. This cuts down on the per-job setup
# time the actual testing jobs have to perform - by perhaps 10% - 20%.
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- image: "docker:17.05.0-ce-git"
steps:
- "checkout"
- "setup_remote_docker"
- run:
name: "Build Debian 9 image"
command: |
docker build -t tahoe-lafs/ci-debian-9:latest -f ~/project/.circleci/Dockerfile.debian-9 .
- run:
name: "Push Debian 9 image"
command: |
# If you create an encryption key like this:
#
# openssl enc -aes-256-cbc -k secret -P -md sha1
# From the output that looks like:
#
# salt=...
# key=...
# iv =...
#
# extract just the value for ``key``.
# then you can re-generate ``secret-env-cipher`` locally using the
# command:
#
# openssl aes-256-cbc -e -in secret-env-plain -out .circleci/secret-env-cipher -k $KEY
#
# Make sure the key is set as the KEY environment variable in the
# CircleCI web interface. You can do this by visiting
# <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
# after logging in to CircleCI with an account in the tahoe-lafs
# CircleCI team.
#
# Then you can recover the environment plaintext (for example, to
# change and re-encrypt it) like just like CircleCI recovers it
# here:
#
openssl aes-256-cbc -d -in .circleci/secret-env-cipher -k $KEY >> ~/secret-environment
# Now get it into the process environment.
. ~/secret-environment
# And clean(-ish) it off the filesystem.
rm ~/secret-environment
docker login -u ${TAHOELAFSCI_USERNAME} -p ${TAHOELAFSCI_PASSWORD}
docker push ${TAHOELAFSCI_USERNAME}/debian:9