tahoe-lafs/.circleci/config.yml

552 lines
14 KiB
YAML
Raw Normal View History

# https://circleci.com/docs/2.0/
version: 2
workflows:
version: 2
ci:
jobs:
# Platforms
- "debian-9"
- "debian-8":
requires:
- "debian-9"
2020-08-07 18:59:27 +00:00
- "ubuntu-20.04"
- "ubuntu-18.04":
requires:
- "ubuntu-20.04"
- "ubuntu-16.04":
requires:
2020-08-07 18:59:27 +00:00
- "ubuntu-20.04"
- "fedora-29"
- "fedora-28":
requires:
- "fedora-29"
- "centos-8"
- "nixos-19.09"
2019-12-13 15:39:15 +00:00
# Test against PyPy 2.7
- "pypy2.7-buster"
# Just one Python 3.6 configuration while the port is in-progress.
- "python3.6"
# Other assorted tasks and configurations
- "lint"
- "pyinstaller"
- "deprecations"
- "c-locale"
# Any locale other than C or UTF-8.
- "another-locale"
- "integration":
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
- "debian-9"
# Generate the underlying data for a visualization to aid with Python 3
# porting.
- "build-porting-depgraph"
2019-04-04 20:54:22 +00:00
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
# faster and takes various spurious failures out of the critical path.
triggers:
# Build once a day
- schedule:
cron: "0 0 * * *"
2019-04-04 21:05:41 +00:00
filters:
branches:
only:
- "master"
2019-04-04 20:54:22 +00:00
jobs:
- "build-image-debian-8"
- "build-image-debian-9"
- "build-image-ubuntu-16.04"
- "build-image-ubuntu-18.04"
2020-08-07 18:59:27 +00:00
- "build-image-ubuntu-20.04"
2019-04-04 20:54:22 +00:00
- "build-image-fedora-28"
- "build-image-fedora-29"
2020-04-13 14:09:25 +00:00
- "build-image-centos-8"
- "build-image-pypy-2.7-buster"
- "build-image-python36-ubuntu"
2019-04-04 20:54:22 +00:00
2018-06-15 12:57:50 +00:00
jobs:
lint:
docker:
- image: "circleci/python:2"
steps:
- "checkout"
- run:
name: "Install tox"
command: |
2018-06-08 18:47:00 +00:00
pip install --user tox
- run:
name: "Static-ish code checks"
command: |
2018-06-08 18:47:00 +00:00
~/.local/bin/tox -e codechecks
2018-06-08 18:58:25 +00:00
pyinstaller:
docker:
- image: "circleci/python:2"
steps:
- "checkout"
- run:
name: "Install tox"
command: |
pip install --user tox
- run:
name: "Make PyInstaller executable"
command: |
~/.local/bin/tox -e pyinstaller
- run:
# To verify that the resultant PyInstaller-generated binary executes
# cleanly (i.e., that it terminates with an exit code of 0 and isn't
# failing due to import/packaging-related errors, etc.).
name: "Test PyInstaller executable"
command: |
dist/Tahoe-LAFS/tahoe --version
debian-9: &DEBIAN
2018-06-08 18:58:25 +00:00
docker:
- image: "tahoelafsci/debian:9-py2.7"
2019-04-04 16:52:35 +00:00
user: "nobody"
2018-06-08 18:58:25 +00:00
environment: &UTF_8_ENVIRONMENT
# In general, the test suite is not allowed to fail while the job
# succeeds. But you can set this to "yes" if you want it to be
# otherwise.
ALLOWED_FAILURE: "no"
# Tell Hypothesis which configuration we want it to use.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
# Tell the C runtime things about character encoding (mainly to do with
2018-06-14 16:32:49 +00:00
# filenames and argv).
LANG: "en_US.UTF-8"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
2018-06-15 17:34:17 +00:00
# Additional arguments to pass to tox.
2018-07-09 19:55:32 +00:00
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
# Convince all of our pip invocations to look at the cached wheelhouse
# we maintain.
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
2018-06-13 16:36:51 +00:00
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
2018-06-08 18:58:25 +00:00
steps:
- "checkout"
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
2018-06-15 19:14:55 +00:00
command: |
2019-04-04 16:52:35 +00:00
/tmp/project/.circleci/setup-virtualenv.sh \
"/tmp/venv" \
"/tmp/project" \
"${WHEELHOUSE_PATH}" \
2018-07-06 15:32:12 +00:00
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
2018-06-08 18:58:25 +00:00
- run: &RUN_TESTS
2018-06-08 18:58:25 +00:00
name: "Run test suite"
2018-06-15 19:14:55 +00:00
command: |
/tmp/project/.circleci/run-tests.sh \
"/tmp/venv" \
"/tmp/project" \
"${ALLOWED_FAILURE}" \
"${ARTIFACTS_OUTPUT_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
2018-07-08 23:06:34 +00:00
# trial output gets directed straight to a log. avoid the circleci
# timeout while the test suite runs.
no_output_timeout: "20m"
- store_test_results: &STORE_TEST_RESULTS
path: "/tmp/artifacts/junit"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
# in the project source checkout.
path: "/tmp/project/_trial_temp/test.log"
- store_artifacts: &STORE_OTHER_ARTIFACTS
# Store any other artifacts, too. This is handy to allow other jobs
# sharing most of the definition of this one to be able to
# contribute artifacts easily.
path: "/tmp/artifacts"
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
command: |
2019-04-04 16:24:16 +00:00
/tmp/venv/bin/codecov
2018-06-14 15:43:59 +00:00
debian-8:
<<: *DEBIAN
2018-06-08 19:52:29 +00:00
docker:
- image: "tahoelafsci/debian:8-py2.7"
user: "nobody"
2019-06-25 17:12:57 +00:00
pypy2.7-buster:
2019-06-25 17:12:57 +00:00
<<: *DEBIAN
docker:
- image: "tahoelafsci/pypy:buster-py2"
2019-06-25 17:12:57 +00:00
user: "nobody"
2018-06-08 19:52:29 +00:00
environment:
<<: *UTF_8_ENVIRONMENT
# We don't do coverage since it makes PyPy far too slow:
TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
2018-06-13 17:59:52 +00:00
2018-06-19 12:37:18 +00:00
c-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
LANG: "C"
2019-03-20 19:56:01 +00:00
another-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
# aka "Latin 1"
LANG: "en_US.ISO-8859-1"
2018-06-15 12:57:50 +00:00
deprecations:
<<: *DEBIAN
2018-06-15 12:57:50 +00:00
environment:
<<: *UTF_8_ENVIRONMENT
# Select the deprecations tox environments.
2018-06-15 12:57:50 +00:00
TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
# Put the logs somewhere we can report them.
TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
2018-06-15 12:57:50 +00:00
integration:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
# Select the integration tests tox environments.
TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
steps:
- "checkout"
2019-04-03 18:27:45 +00:00
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
2018-06-14 15:44:39 +00:00
ubuntu-16.04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:16.04-py2.7"
user: "nobody"
2018-06-14 16:32:49 +00:00
2018-06-14 15:44:39 +00:00
ubuntu-18.04: &UBUNTU_18_04
2018-06-14 14:46:45 +00:00
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:18.04-py2.7"
user: "nobody"
2018-06-14 14:46:45 +00:00
python3.6:
<<: *UBUNTU_18_04
docker:
- image: "tahoelafsci/ubuntu:18.04-py3"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# The default trial args include --rterrors which is incompatible with
# this reporter on Python 3. So drop that and just specify the
# reporter.
TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
2020-08-07 18:59:27 +00:00
ubuntu-20.04:
<<: *DEBIAN
docker:
- image: "tahoelafsci/ubuntu:20.04"
user: "nobody"
2020-04-13 14:09:25 +00:00
centos-8: &RHEL_DERIV
2018-06-13 17:59:52 +00:00
docker:
- image: "tahoelafsci/centos:8-py2"
2019-04-04 18:00:19 +00:00
user: "nobody"
2018-06-13 17:59:52 +00:00
2018-06-14 16:32:49 +00:00
environment: *UTF_8_ENVIRONMENT
2018-06-13 17:59:52 +00:00
2019-04-04 18:00:19 +00:00
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
2018-06-13 17:59:52 +00:00
2019-04-04 18:00:19 +00:00
steps:
2018-06-13 17:59:52 +00:00
- "checkout"
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
fedora-28:
<<: *RHEL_DERIV
2018-06-14 14:09:42 +00:00
docker:
- image: "tahoelafsci/fedora:28-py"
user: "nobody"
2018-06-14 14:09:42 +00:00
2018-06-14 14:11:49 +00:00
fedora-29:
<<: *RHEL_DERIV
2018-06-14 14:11:49 +00:00
docker:
- image: "tahoelafsci/fedora:29-py"
user: "nobody"
2018-06-15 17:34:17 +00:00
2019-12-13 15:39:15 +00:00
nixos-19.09:
docker:
# Run in a highly Nix-capable environment.
- image: "nixorg/nix:circleci"
environment:
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
steps:
- "checkout"
- "run":
name: "Build and Test"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suites the build environment we're paying
# for (the free one!).
#
# Also, let it run more than one job at a time because we have to
# build a couple simple little dependencies that don't take
# advantage of multiple cores and we get a little speedup by doing
# them in parallel.
nix-build --cores 3 --max-jobs 2 nix/
2019-12-13 15:39:15 +00:00
2019-09-13 17:00:35 +00:00
# Generate up-to-date data for the dependency graph visualizer.
build-porting-depgraph:
2019-09-13 17:00:35 +00:00
# Get a system in which we can easily install Tahoe-LAFS and all its
# dependencies. The dependency graph analyzer works by executing the code.
# It's Python, what do you expect?
<<: *DEBIAN
steps:
- "checkout"
2019-09-13 17:00:35 +00:00
- add_ssh_keys:
fingerprints:
# Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
# This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
- "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
2019-09-13 17:00:35 +00:00
- run:
name: "Setup Python Environment"
command: |
/tmp/venv/bin/pip install -e /tmp/project
- run:
name: "Generate dependency graph data"
command: |
. /tmp/venv/bin/activate
./misc/python3/depgraph.sh
2019-04-04 19:34:34 +00:00
build-image: &BUILD_IMAGE
# This is a template for a job to build a Docker image that has as much of
# the setup as we can manage already done and baked in. This cuts down on
# the per-job setup time the actual testing jobs have to perform - by
# perhaps 10% - 20%.
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- image: "docker:17.05.0-ce-git"
2019-04-04 19:34:34 +00:00
environment:
DISTRO: "tahoelafsci/<DISTRO>:foo-py2"
TAG: "tahoelafsci/distro:<TAG>-py2"
PYTHON_VERSION: "tahoelafsci/distro:tag-py<PYTHON_VERSION}"
2019-04-04 19:34:34 +00:00
steps:
- "checkout"
- "setup_remote_docker"
2019-04-03 17:41:33 +00:00
- run:
name: "Get openssl"
command: |
2019-04-03 17:44:00 +00:00
apk add --no-cache openssl
- run:
name: "Get Dockerhub secrets"
command: |
# If you create an encryption key like this:
#
2019-04-03 18:00:26 +00:00
# openssl enc -aes-256-cbc -k secret -P -md sha256
# From the output that looks like:
#
# salt=...
# key=...
# iv =...
#
# extract just the value for ``key``.
# then you can re-generate ``secret-env-cipher`` locally using the
# command:
#
# openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
#
# Make sure the key is set as the KEY environment variable in the
# CircleCI web interface. You can do this by visiting
# <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
# after logging in to CircleCI with an account in the tahoe-lafs
# CircleCI team.
#
# Then you can recover the environment plaintext (for example, to
# change and re-encrypt it) like just like CircleCI recovers it
# here:
#
2019-04-03 19:37:21 +00:00
openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
- run:
name: "Log in to Dockerhub"
command: |
2019-04-03 19:37:21 +00:00
. ~/.env
# TAHOELAFSCI_PASSWORD come from the secret env.
docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
- run:
2019-04-04 19:34:34 +00:00
name: "Build image"
command: |
2019-04-04 19:34:34 +00:00
docker \
build \
--build-arg TAG=${TAG} \
2020-07-17 20:26:56 +00:00
--build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-t tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} \
2019-04-04 20:51:51 +00:00
-f ~/project/.circleci/Dockerfile.${DISTRO} \
2019-04-04 19:34:34 +00:00
~/project/
2019-04-04 19:41:22 +00:00
- run:
name: "Push image"
command: |
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
2019-04-04 19:34:34 +00:00
build-image-debian-8:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "8"
PYTHON_VERSION: "2.7"
2019-04-04 19:34:34 +00:00
build-image-debian-9:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "9"
PYTHON_VERSION: "2.7"
2019-04-04 19:34:34 +00:00
build-image-ubuntu-16.04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "16.04"
PYTHON_VERSION: "2.7"
2019-04-04 19:34:34 +00:00
2019-04-04 19:59:58 +00:00
build-image-ubuntu-18.04:
2019-04-04 19:34:34 +00:00
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
PYTHON_VERSION: "2.7"
build-image-python36-ubuntu:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
PYTHON_VERSION: "3"
2019-04-04 19:34:34 +00:00
2020-08-07 18:59:27 +00:00
build-image-ubuntu-20.04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "20.04"
PYTHON_VERSION: "2.7"
2020-08-07 18:59:27 +00:00
2020-04-13 14:09:25 +00:00
build-image-centos-8:
2019-04-04 19:34:34 +00:00
<<: *BUILD_IMAGE
environment:
DISTRO: "centos"
2020-04-13 14:09:25 +00:00
TAG: "8"
PYTHON_VERSION: "2"
2019-04-04 19:34:34 +00:00
build-image-fedora-28:
<<: *BUILD_IMAGE
environment:
DISTRO: "fedora"
TAG: "28"
# The default on Fedora (this version anyway) is still Python 2.
PYTHON_VERSION: ""
2019-04-04 19:34:34 +00:00
build-image-fedora-29:
<<: *BUILD_IMAGE
environment:
DISTRO: "fedora"
TAG: "29"
2019-04-04 20:50:59 +00:00
build-image-pypy-2.7-buster:
<<: *BUILD_IMAGE
environment:
DISTRO: "pypy"
TAG: "buster"
# We only have Python 2 for PyPy right now so there's no support for
# setting up PyPy 3 in the image building toolchain. This value is just
# for constructing the right Docker image tag.
PYTHON_VERSION: "2"