ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-21 13:57:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
4c9321eff1
tahoe-lafs/.circleci/config.yml
meejah 4c9321eff1 see if wheel cache works now
2024-12-06 13:20:18 -07:00

755 lines
24 KiB
YAML
Raw Blame History

# https://circleci.com/docs/2.0/
# We use version 2.1 of CircleCI's configuration format (the docs are still at
# the 2.0 link) in order to have access to Windows executors. This means we
# can't use dots in job names anymore. They have a new "parameters" feature
# that is supposed to remove the need to have version numbers in job names (the
# source of our dots), but switching to that is going to be a bigger refactor:
#
# https://discuss.circleci.com/t/v2-1-job-name-validation/31123
# https://circleci.com/docs/2.0/reusing-config/
#
version: 2.1
# Every job that pushes a Docker image from Docker Hub must authenticate to
# it. Define a couple yaml anchors that can be used to supply the necessary
# credentials.
# First is a CircleCI job context which makes Docker Hub credentials available
# in the environment.
#
# Contexts are managed in the CircleCI web interface:
#
# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
dockerhub-context-template: &DOCKERHUB_CONTEXT
context: "dockerhub-auth"
# Required environment for using the coveralls tool to upload partial coverage
# reports and then finish the process.
coveralls-environment: &COVERALLS_ENVIRONMENT
COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
# Next is a Docker executor template that gets the credentials from the
# environment and supplies them to the executor.
dockerhub-auth-template: &DOCKERHUB_AUTH
- auth:
username: $DOCKERHUB_USERNAME
password: $DOCKERHUB_PASSWORD
# A template that can be shared between the two different image-building
# workflows.
.images: &IMAGES
jobs:
- "build-image-debian-11":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-20-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-ubuntu-22-04":
<<: *DOCKERHUB_CONTEXT
- "build-image-fedora-35":
<<: *DOCKERHUB_CONTEXT
# Restore later as PyPy38
#- "build-image-pypy27-buster":
# <<: *DOCKERHUB_CONTEXT
parameters:
# Control whether the image-building workflow runs as part of this pipeline.
# Generally we do not want this to run because we don't need our
# dependencies to move around all the time and because building the image
# takes a couple minutes.
#
# An easy way to trigger a pipeline with this set to true is with the
# rebuild-images.sh tool in this directory. You can also do so via the
# CircleCI web UI.
build-images:
default: false
type: "boolean"
# Control whether the test-running workflow runs as part of this pipeline.
# Generally we do want this to run because running the tests is the primary
# purpose of this pipeline.
run-tests:
default: true
type: "boolean"
workflows:
ci:
when: "<< pipeline.parameters.run-tests >>"
jobs:
# Start with jobs testing various platforms.
- "debian-11":
{}
- "ubuntu-20-04":
{}
- "ubuntu-22-04":
{}
- "nixos":
name: "<<matrix.pythonVersion>>"
nixpkgs: "nixpkgs-unstable"
matrix:
parameters:
pythonVersion:
- "python39"
- "python310"
- "python311"
# Eventually, test against PyPy 3.8
#- "pypy27-buster":
# {}
# Other assorted tasks and configurations
- "codechecks":
{}
- "pyinstaller":
{}
- "c-locale":
{}
# Any locale other than C or UTF-8.
- "another-locale":
{}
- "windows-server-2022":
name: "Windows Server 2022, CPython <<matrix.pythonVersion>>"
matrix:
parameters:
# Run the job for a number of CPython versions. These are the
# two versions installed on the version of the Windows VM image
# we specify (in the executor). This is handy since it means we
# don't have to do any Python installation work. We pin the
# Windows VM image so these shouldn't shuffle around beneath us
# but if we want to update that image or get different versions
# of Python, we probably have to do something here.
pythonVersion:
- "3.9"
- "3.12"
- "integration":
# Run even the slow integration tests here. We need the `--` to
# sneak past tox and get to pytest.
tox-args: "-- --runslow integration"
requires:
# If the unit test suite doesn't pass, don't bother running the
# integration tests.
- "debian-11"
- "typechecks":
{}
- "docs":
{}
- "finish-coverage-report":
requires:
# Referencing the job by "alias" (as CircleCI calls the mapping
# key) instead of the value of its "name" property causes us to
# require every instance of the job from its matrix expansion. So
# this requirement is enough to require every Windows Server 2022
# job.
- "windows-server-2022"
images:
<<: *IMAGES
# Build as part of the workflow but only if requested.
when: "<< pipeline.parameters.build-images >>"
jobs:
finish-coverage-report:
docker:
- <<: *DOCKERHUB_AUTH
image: "python:3-slim"
steps:
- run:
name: "Indicate completion to coveralls.io"
environment:
<<: *COVERALLS_ENVIRONMENT
command: |
pip install coveralls==3.3.1
python -m coveralls --finish
codechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "cimg/python:3.9"
steps:
- "checkout"
- run: &INSTALL_TOX
name: "Install tox"
command: |
pip install --user 'tox~=3.0'
- run:
name: "Static-ish code checks"
command: |
~/.local/bin/tox -e codechecks
windows-server-2022:
parameters:
pythonVersion:
description: >-
An argument to pass to the `py` launcher to choose a Python version.
type: "string"
default: ""
executor: "windows"
environment:
# Tweak Hypothesis to make its behavior more suitable for the CI
# environment. This should improve reproducibility and lessen the
# effects of variable compute resources.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
# Tell pip where its download cache lives. This must agree with the
# "save_cache" step below or caching won't really work right.
PIP_CACHE_DIR: "pip-cache"
# And tell pip where it can find out cached wheelhouse for fast wheel
# installation, even for projects that don't distribute wheels. This
# must also agree with the "save_cache" step below.
PIP_FIND_LINKS: "wheelhouse"
steps:
- "checkout"
# If possible, restore a pip download cache to save us from having to
# download all our Python dependencies from PyPI.
- "restore_cache":
keys:
# The download cache and/or the wheelhouse may contain Python
# version-specific binary packages so include the Python version
# in this key, as well as the canonical source of our
# dependencies.
- &CACHE_KEY "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"pyproject.toml\" }}"
- "run":
name: "Fix $env:PATH"
command: |
# The Python this job is parameterized is not necessarily the one
# at the front of $env:PATH. Modify $env:PATH so that it is so we
# can just say "python" in the rest of the steps. Also get the
# related Scripts directory so tools from packages we install are
# also available.
$p = py -<<parameters.pythonVersion>> -c "import sys; print(sys.prefix)"
$q = py -<<parameters.pythonVersion>> -c "import sysconfig; print(sysconfig.get_path('scripts'))"
New-Item $Profile.CurrentUserAllHosts -Force
# $p gets "python" on PATH and $q gets tools from packages we
# install. Note we carefully construct the string so that
# $env:PATH is not substituted now but $p and $q are. ` is the
# PowerShell string escape character.
Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`""
- "run":
# It's faster to install a wheel than a source package. If we don't
# have a cached wheelhouse then build all of the wheels and dump
# them into a directory where they can become a cached wheelhouse.
# We would have built these wheels during installation anyway so it
# doesn't cost us anything extra and saves us effort next time.
name: "(Maybe) Build Wheels"
command: |
if ((Test-Path .\wheelhouse) -and (Test-Path .\wheelhouse\*)) {
echo "Found populated wheelhouse, skipping wheel building."
} else {
python -m pip install wheel
python -m pip wheel --wheel-dir $env:PIP_FIND_LINKS .[testenv] .[test]
}
- "save_cache":
paths:
# Make sure this agrees with PIP_CACHE_DIR in the environment.
- "pip-cache"
- "wheelhouse"
key: *CACHE_KEY
- "run":
name: "Install Dependencies"
environment:
# By this point we should no longer need an index.
## PIP_NO_INDEX: "1"
command: |
python -m pip install .[testenv] .[test]
- "run":
name: "Display tool versions"
command: |
python misc/build_helpers/show-tool-versions.py
- "run":
name: "Run Unit Tests"
environment:
# Configure the results location for the subunitv2-file reporter
# from subunitreporter
SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2"
# Try to get prompt output from the reporter to avoid no-output
# timeouts.
PYTHONUNBUFFERED: "1"
command: |
# Run the test suite under coverage measurement using the
# parameterized version of Python, writing subunitv2-format
# results to the file given in the environment.
python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata
- "run":
name: "Upload Coverage"
environment:
<<: *COVERALLS_ENVIRONMENT
# Mark the data as just one piece of many because we have more
# than one instance of this job (two on Windows now, some on other
# platforms later) which collects and reports coverage. This is
# necessary to cause Coveralls to merge multiple coverage results
# into a single report. Note the merge only happens when we
# "finish" a particular build, as identified by its "build_num"
# (aka "service_number").
COVERALLS_PARALLEL: "true"
command: |
python -m pip install coveralls==3.3.1
# .coveragerc sets parallel = True so we don't have a `.coverage`
# file but a `.coverage.<unique stuff>` file (or maybe more than
# one, but probably not). coveralls can't work with these so
# merge them before invoking it.
python -m coverage combine
# Now coveralls will be able to find the data, so have it do the
# upload. Also, have it strip the system config-specific prefix
# from all of the source paths.
$prefix = python -c "import sysconfig; print(sysconfig.get_path('purelib'))"
python -m coveralls --basedir $prefix
- "run":
name: "Convert Result Log"
command: |
# subunit2junitxml exits with error if the result stream it is
# converting has test failures in it! So this step might fail.
# Since the step in which we actually _ran_ the tests won't fail
# even if there are test failures, this is a good thing for now.
subunit2junitxml.exe --output-to=test-results.xml test-results.subunit2
- "store_test_results":
path: "test-results.xml"
- "store_artifacts":
path: "_trial_temp/test.log"
- "store_artifacts":
path: "eliot.log"
- "store_artifacts":
path: ".coverage"
pyinstaller:
docker:
- <<: *DOCKERHUB_AUTH
image: "cimg/python:3.9"
steps:
- "checkout"
- run:
<<: *INSTALL_TOX
- run:
name: "Make PyInstaller executable"
command: |
~/.local/bin/tox -e pyinstaller
- run:
# To verify that the resultant PyInstaller-generated binary executes
# cleanly (i.e., that it terminates with an exit code of 0 and isn't
# failing due to import/packaging-related errors, etc.).
name: "Test PyInstaller executable"
command: |
dist/Tahoe-LAFS/tahoe --version
debian-11: &DEBIAN
environment: &UTF_8_ENVIRONMENT
# In general, the test suite is not allowed to fail while the job
# succeeds. But you can set this to "yes" if you want it to be
# otherwise.
ALLOWED_FAILURE: "no"
# Tell Hypothesis which configuration we want it to use.
TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
# Tell the C runtime things about character encoding (mainly to do with
# filenames and argv).
LANG: "en_US.UTF-8"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
# Additional arguments to pass to tox.
TAHOE_LAFS_TOX_ARGS: ""
# The path in which test artifacts will be placed.
ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
# Convince all of our pip invocations to look at the cached wheelhouse
# we maintain.
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
# Upload the coverage report.
UPLOAD_COVERAGE: ""
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
steps:
- "checkout"
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
command: |
/tmp/project/.circleci/setup-virtualenv.sh \
"/tmp/venv" \
"/tmp/project" \
"${WHEELHOUSE_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
- run: &RUN_TESTS
name: "Run test suite"
command: |
/tmp/project/.circleci/run-tests.sh \
"/tmp/venv" \
"/tmp/project" \
"${ALLOWED_FAILURE}" \
"${ARTIFACTS_OUTPUT_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
# trial output gets directed straight to a log. avoid the circleci
# timeout while the test suite runs.
no_output_timeout: "20m"
- store_test_results: &STORE_TEST_RESULTS
path: "/tmp/artifacts/junit"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
# in the project source checkout.
path: "/tmp/project/_trial_temp/test.log"
- store_artifacts: &STORE_ELIOT_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
# in the project source checkout.
path: "/tmp/project/eliot.log"
- store_artifacts: &STORE_OTHER_ARTIFACTS
# Store any other artifacts, too. This is handy to allow other jobs
# sharing most of the definition of this one to be able to
# contribute artifacts easily.
path: "/tmp/artifacts"
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
command: |
if [ -n "${UPLOAD_COVERAGE}" ]; then
echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011"
fi
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
user: "nobody"
# Restore later using PyPy3.8
# pypy27-buster:
# <<: *DEBIAN
# docker:
# - <<: *DOCKERHUB_AUTH
# image: "tahoelafsci/pypy:buster-py2"
# user: "nobody"
# environment:
# <<: *UTF_8_ENVIRONMENT
# # We don't do coverage since it makes PyPy far too slow:
# TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
# # Since we didn't collect it, don't upload it.
# UPLOAD_COVERAGE: ""
c-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
LANG: "C"
another-locale:
<<: *DEBIAN
environment:
<<: *UTF_8_ENVIRONMENT
# aka "Latin 1"
LANG: "en_US.ISO-8859-1"
integration:
<<: *DEBIAN
parameters:
tox-args:
description: >-
Additional arguments to pass to the tox command.
type: "string"
default: ""
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/debian:11-py3.9"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
# Select the integration tests tox environments.
TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
# Pass on anything we got in our parameters.
TAHOE_LAFS_TOX_ARGS: "<< parameters.tox-args >>"
steps:
- "checkout"
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
ubuntu-20-04:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:20.04-py3.9"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
ubuntu-22-04:
<<: *DEBIAN
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:22.04-py3.10"
user: "nobody"
environment:
<<: *UTF_8_ENVIRONMENT
TAHOE_LAFS_TOX_ENVIRONMENT: "py310"
nixos:
parameters:
nixpkgs:
description: >-
Reference the name of a flake-managed nixpkgs input (see `nix flake
metadata` and flake.nix)
type: "string"
pythonVersion:
description: >-
Reference the name of a Python package in nixpkgs to use.
type: "string"
executor: "nix"
steps:
- "nix-build":
nixpkgs: "<<parameters.nixpkgs>>"
pythonVersion: "<<parameters.pythonVersion>>"
buildSteps:
- "run":
name: "Unit Test"
command: |
source .circleci/lib.sh
# Translate the nixpkgs selection into a flake reference we
# can use to override the default nixpkgs input.
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix run \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-unittest -- \
--jobs $UNITTEST_CORES \
allmydata
typechecks:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
- run:
name: "Validate Types"
command: |
/tmp/venv/bin/tox -e typechecks
docs:
docker:
- <<: *DOCKERHUB_AUTH
image: "tahoelafsci/ubuntu:20.04-py3.9"
steps:
- "checkout"
- run:
name: "Build documentation"
command: |
/tmp/venv/bin/tox -e docs
build-image: &BUILD_IMAGE
# This is a template for a job to build a Docker image that has as much of
# the setup as we can manage already done and baked in. This cuts down on
# the per-job setup time the actual testing jobs have to perform - by
# perhaps 10% - 20%.
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- <<: *DOCKERHUB_AUTH
# CircleCI build images; https://github.com/CircleCI-Public/cimg-base
# for details.
image: "cimg/base:2022.01"
environment:
DISTRO: "tahoelafsci/<DISTRO>:foo-py3.9"
TAG: "tahoelafsci/distro:<TAG>-py3.9"
PYTHON_VERSION: "tahoelafsci/distro:tag-py<PYTHON_VERSION}"
steps:
- "checkout"
- setup_remote_docker:
version: "20.10.11"
- run:
name: "Log in to Dockerhub"
command: |
docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD}
- run:
name: "Build image"
command: |
docker \
build \
--build-arg TAG=${TAG} \
--build-arg PYTHON_VERSION=${PYTHON_VERSION} \
-t tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION} \
-f ~/project/.circleci/Dockerfile.${DISTRO} \
~/project/
- run:
name: "Push image"
command: |
docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
build-image-debian-11:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "11"
PYTHON_VERSION: "3.9"
build-image-ubuntu-20-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "20.04"
PYTHON_VERSION: "3.9"
build-image-ubuntu-22-04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "22.04"
PYTHON_VERSION: "3.10"
build-image-fedora-35:
<<: *BUILD_IMAGE
environment:
DISTRO: "fedora"
TAG: "35"
PYTHON_VERSION: "3"
# build-image-pypy27-buster:
# <<: *BUILD_IMAGE
# environment:
# DISTRO: "pypy"
# TAG: "buster"
# # We only have Python 2 for PyPy right now so there's no support for
# # setting up PyPy 3 in the image building toolchain. This value is just
# # for constructing the right Docker image tag.
# PYTHON_VERSION: "2"
executors:
windows:
# Choose a Windows environment that closest matches our testing
# requirements and goals.
# https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022
machine:
image: "windows-server-2022-gui:current"
shell: "powershell.exe -ExecutionPolicy Bypass"
resource_class: "windows.large"
nix:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.16.1"
environment:
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
# to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push
# to.
CACHIX_NAME: "tahoe-lafs-opensource"
# Let us use features marked "experimental". For example, most/all of
# the `nix <subcommand>` forms.
NIX_CONFIG: "experimental-features = nix-command flakes"
commands:
nix-build:
parameters:
nixpkgs:
description: >-
Reference the name of a flake-managed nixpkgs input (see `nix flake
metadata` and flake.nix)
type: "string"
pythonVersion:
description: >-
Reference the name of a Python package in nixpkgs to use.
type: "string"
buildSteps:
description: >-
The build steps to execute after setting up the build environment.
type: "steps"
steps:
- "run":
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
# Get some build environment dependencies and let them float on a
# certain release branch. These aren't involved in the actual
# package build (only in CI environment setup) so the fact that
# they float shouldn't hurt reproducibility.
NIXPKGS="nixpkgs/nixos-23.05"
nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp
# Activate our cachix cache for "binary substitution". This sets
# up configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"
- "run":
name: "Build Package"
command: |
source .circleci/lib.sh
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix build \
--verbose \
--print-build-logs \
--cores "$DEPENDENCY_CORES" \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-tahoe-lafs
- steps: "<<parameters.buildSteps>>"
Reference in New Issue View Git Blame Copy Permalink