ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-21 02:01:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #595 from tahoe-lafs/3024.faster-circleci-builds

Browse Source 
Faster CircleCI builds

Fixes: ticket:3024
This commit is contained in:
Jean-Paul Calderone 2019-04-08 17:32:20 -04:00 committed by GitHub
commit 325c522d7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 578 additions and 305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.circleci/Dockerfile.centos Normal file
View File

@ -0,0 +1,26 @@
ARG TAG
FROM centos:${TAG}
ENV WHEELHOUSE_PATH /tmp/wheelhouse
ENV VIRTUALENV_PATH /tmp/venv
# This will get updated by the CircleCI checkout step.
ENV BUILD_SRC_ROOT /tmp/project
# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
RUN yum install --assumeyes \
git \
sudo \
make automake gcc gcc-c++ \
python \
python-devel \
libffi-devel \
openssl-devel \
libyaml-devel \
/usr/bin/virtualenv \
net-tools
# Get the project source. This is better than it seems. CircleCI will
# *update* this checkout on each job run, saving us more time per-job.
COPY . ${BUILD_SRC_ROOT}
RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

31
.circleci/Dockerfile.debian Normal file
View File

@ -0,0 +1,31 @@
ARG TAG
FROM debian:${TAG}
ENV WHEELHOUSE_PATH /tmp/wheelhouse
ENV VIRTUALENV_PATH /tmp/venv
# This will get updated by the CircleCI checkout step.
ENV BUILD_SRC_ROOT /tmp/project
RUN apt-get --quiet update && \
apt-get --quiet --yes install \
git \
lsb-release \
sudo \
build-essential \
python2.7 \
python2.7-dev \
libffi-dev \
libssl-dev \
libyaml-dev \
virtualenv
# Get the project source. This is better than it seems. CircleCI will
# *update* this checkout on each job run, saving us more time per-job.
COPY . ${BUILD_SRC_ROOT}
RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
# Only the integration tests currently need this but it doesn't hurt to always
# have it present and it's simpler than building a whole extra image just for
# the integration tests.
RUN ${BUILD_SRC_ROOT}/integration/install-tor.sh

26
.circleci/Dockerfile.fedora Normal file
View File

@ -0,0 +1,26 @@
ARG TAG
FROM fedora:${TAG}
ENV WHEELHOUSE_PATH /tmp/wheelhouse
ENV VIRTUALENV_PATH /tmp/venv
# This will get updated by the CircleCI checkout step.
ENV BUILD_SRC_ROOT /tmp/project
# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
RUN yum install --assumeyes \
git \
sudo \
make automake gcc gcc-c++ \
python \
python-devel \
libffi-devel \
openssl-devel \
libyaml-devel \
/usr/bin/virtualenv \
net-tools
# Get the project source. This is better than it seems. CircleCI will
# *update* this checkout on each job run, saving us more time per-job.
COPY . ${BUILD_SRC_ROOT}
RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

49
.circleci/Dockerfile.slackware Normal file
View File

@ -0,0 +1,49 @@
ARG TAG
FROM vbatts/slackware:${TAG}
ENV WHEELHOUSE_PATH /tmp/wheelhouse
ENV VIRTUALENV_PATH /tmp/venv
# This will get updated by the CircleCI checkout step.
ENV BUILD_SRC_ROOT /tmp/project
# Be careful with slackpkg. If the package name given doesn't match anything,
# slackpkg still claims to succeed but you're totally screwed. Slackware
# updates versions of packaged software so including too much version prefix
# is a good way to have your install commands suddenly begin not installing
# anything.
RUN slackpkg update && \
slackpkg install \
openssh-7 git-2 \
ca-certificates \
sudo-1 \
make-4 \
automake-1 \
kernel-headers \
glibc-2 \
binutils-2 \
gcc-5 \
gcc-g++-5 \
python-2 \
libffi-3 \
libyaml-0 \
sqlite-3 \
icu4c-56 \
libmpc-1 </dev/null && \
slackpkg upgrade \
openssl-1 </dev/null
# neither virtualenv nor pip is packaged.
# do it the hard way.
# and it is extra hard since it is slackware.
RUN slackpkg install \
cyrus-sasl-2 \
curl-7 </dev/null && \
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python get-pip.py && \
pip install virtualenv
# Get the project source. This is better than it seems. CircleCI will
# *update* this checkout on each job run, saving us more time per-job.
COPY . ${BUILD_SRC_ROOT}
RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

29
.circleci/Dockerfile.ubuntu Normal file
View File

@ -0,0 +1,29 @@
ARG TAG
FROM ubuntu:${TAG}
ENV WHEELHOUSE_PATH /tmp/wheelhouse
ENV VIRTUALENV_PATH /tmp/venv
# This will get updated by the CircleCI checkout step.
ENV BUILD_SRC_ROOT /tmp/project
# language-pack-en included to support the en_US LANG setting.
# iproute2 necessary for automatic address detection/assignment.
RUN apt-get --quiet update && \
apt-get --quiet --yes install git && \
apt-get --quiet --yes install \
sudo \
build-essential \
python2.7 \
python2.7-dev \
libffi-dev \
libssl-dev \
libyaml-dev \
virtualenv \
language-pack-en \
iproute2
# Get the project source. This is better than it seems. CircleCI will
# *update* this checkout on each job run, saving us more time per-job.
COPY . ${BUILD_SRC_ROOT}
RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

27
.circleci/bootstrap-test-environment.sh
View File

@ -1,27 +0,0 @@
#!/bin/bash -e
PROJECT=$1
shift
EXTRA_PACKAGES=$1
shift
# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try to test this
# checkout.
mv "${PROJECT}" /tmp/project
# Python build/install toolchain wants to write to the source checkout, too.
chown --recursive nobody:nogroup /tmp/project
apt-get --quiet --yes install \
sudo \
build-essential \
python2.7 \
python2.7-dev \
libffi-dev \
libssl-dev \
libyaml-dev \
${EXTRA_PACKAGES}

403
.circleci/config.yml
View File

@ -38,6 +38,28 @@ workflows:
# integration tests.
- "debian-9"
images:
# Build the Docker images used by the ci jobs. This makes the ci jobs
# faster and takes various spurious failures out of the critical path.
triggers:
# Build once a day
- schedule:
cron: "0 0 * * *"
filters:
branches:
only:
- "master"
jobs:
- "build-image-debian-8"
- "build-image-debian-9"
- "build-image-ubuntu-16.04"
- "build-image-ubuntu-18.04"
- "build-image-fedora-28"
- "build-image-fedora-29"
- "build-image-centos-7"
- "build-image-slackware-14.2"
jobs:
lint:
@ -59,7 +81,8 @@ jobs:
debian-9: &DEBIAN
docker:
- image: "debian:9"
- image: "tahoelafsci/debian:9"
user: "nobody"
environment: &UTF_8_ENVIRONMENT
# Tell Hypothesis which configuration we want it to use.
@ -67,8 +90,6 @@ jobs:
# Tell the C runtime things about character encoding (mainly to do with
# filenames and argv).
LANG: "en_US.UTF-8"
# The package name for this software varies somewhat across distros.
EXTRA_PACKAGES: "virtualenv"
# Select a tox environment to run for this job.
TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
# Additional arguments to pass to tox.
@ -80,82 +101,28 @@ jobs:
WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
PIP_FIND_LINKS: "file:///tmp/wheelhouse"
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
steps:
- run: &INSTALL_GIT
name: "Install Git"
command: |
apt-get --quiet update
apt-get --quiet --yes install git
- "checkout"
- run: &BOOTSTRAP_TEST_ENVIRONMENT
name: "Bootstrap test environment"
command: |
~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
- restore_cache: &RESTORE_HTTP_CACHE
name: "Restoring pip HTTP cache"
keys:
# An exact match on the http cache key is great. It should have
# exactly the packages (tgz, whl, whatever) we need.
- v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A prefix match is okay too. It might have a
# partially-overlapping set of packages. That's a head-start, at
# least. We might have to download a few more things but at least
# we saved a little time. After we download some more stuff we'll
# create a new cache entry with the full key above and the next
# build will get a better cache hit.
- v5-pip-http-
- restore_cache: &RESTORE_WHEELHOUSE
name: "Restoring wheelhouse"
keys:
# As above, an exact match is great. Here, we also need to
# include the job name to make sure the platform ABI matches.
# There are binary wheels in this wheelhouse and we're not taking
# care to make manylinux1 wheels. The binary wheels in this cache
# will only work on some Linux distros.
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
# A partial match is okay too. It'll get us at least some of the
# wheels. We do need to keep the job name as part of the key or
# we might get binary wheels build against an incompatible ABI and
# we won't be able to use them (and they'll break the build rather
# than being ignored).
- v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}
- run: &SETUP_VIRTUALENV
name: "Setup virtualenv"
# pip cannot install packages if the working directory is not
# readable.
working_directory: "/tmp"
command: |
/tmp/project/.circleci/setup-virtualenv.sh \
"/tmp/venv" \
"/tmp/project" \
"${WHEELHOUSE_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
- save_cache: &SAVE_HTTP_CACHE
name: "Saving pip HTTP cache"
key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
# Perfectly valid for Linux. Note we exclude the wheel cache
# because we want this cache to be valid across all platforms and
# the wheels in the pip wheel cache are not necessarily so.
- "/tmp/nobody/.cache/pip/http"
- save_cache: &SAVE_WHEELHOUSE
name: "Caching wheelhouse"
key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
paths:
- *WHEELHOUSE_PATH
- run: &RUN_TESTS
name: "Run test suite"
# Something about when it re-uses an existing environment blows up
# if the working directory is not readable.
working_directory: "/tmp"
command: |
/tmp/project/.circleci/run-tests.sh \
"/tmp/venv" \
"/tmp/project" \
"${ARTIFACTS_OUTPUT_PATH}" \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
@ -179,15 +146,15 @@ jobs:
- run: &SUBMIT_COVERAGE
name: "Submit coverage results"
working_directory: "/tmp/project"
command: |
/tmp/tests/bin/codecov
/tmp/venv/bin/codecov
debian-8:
<<: *DEBIAN
docker:
- image: "debian:8"
- image: "tahoelafsci/debian:8"
user: "nobody"
c-locale:
@ -228,101 +195,42 @@ jobs:
# Disable artifact collection because py.test can't produce any.
ARTIFACTS_OUTPUT_PATH: ""
steps:
# DRY, YAML-style. See the debian-9 steps.
- run: *INSTALL_GIT
- "checkout"
- run: *BOOTSTRAP_TEST_ENVIRONMENT
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
# DRY, YAML-style. See the debian-9 steps.
- run: *SETUP_VIRTUALENV
- run:
name: "Install Tor"
command: |
# Don't forget that we moved the source tree to /tmp.
/tmp/project/integration/install-tor.sh
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
ubuntu-16.04:
<<: *DEBIAN
docker:
- image: "ubuntu:16.04"
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for en_US LANG setting.
EXTRA_PACKAGES: "virtualenv language-pack-en"
- image: "tahoelafsci/ubuntu:16.04"
user: "nobody"
ubuntu-18.04:
<<: *DEBIAN
docker:
- image: "ubuntu:18.04"
environment:
<<: *UTF_8_ENVIRONMENT
# Necessary for automatic address detection/assignment.
EXTRA_PACKAGES: "virtualenv iproute2"
- image: "tahoelafsci/ubuntu:18.04"
user: "nobody"
centos-7: &RHEL_DERIV
docker:
- image: "centos:7"
- image: "tahoelafsci/centos:7"
user: "nobody"
environment: *UTF_8_ENVIRONMENT
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
steps:
- run:
name: "Install Git"
command: |
yum install --assumeyes git
- "checkout"
- run:
name: "Bootstrap test environment"
working_directory: "/tmp"
command: |
# Avoid the /nonexistent home directory in nobody's /etc/passwd
# entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try
# to test this checkout.
mv /root/project /tmp/project
# Python build/install toolchain wants to write to the source
# checkout, too.
chown --recursive nobody:nobody /tmp/project
yum install --assumeyes \
sudo \
make automake gcc gcc-c++ \
python \
python-devel \
libffi-devel \
openssl-devel \
libyaml-devel
yum install --assumeyes /usr/bin/virtualenv
# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
yum install --assumeyes \
net-tools
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
@ -332,94 +240,169 @@ jobs:
fedora-28:
<<: *RHEL_DERIV
docker:
- image: "fedora:28"
- image: "tahoelafsci/fedora:28"
user: "nobody"
fedora-29:
<<: *RHEL_DERIV
docker:
- image: "fedora:29"
- image: "tahoelafsci/fedora:29"
user: "nobody"
slackware-14.2:
docker:
- image: "vbatts/slackware:14.2"
- image: "tahoelafsci/slackware:14.2"
user: "nobody"
environment: *UTF_8_ENVIRONMENT
# pip cannot install packages if the working directory is not readable.
# We want to run a lot of steps as nobody instead of as root.
working_directory: "/tmp/project"
steps:
- run:
name: "Install Git"
command: |
slackpkg update
# Be careful with slackpkg. If the package name given doesn't
# match anything, slackpkg still claims to succeed but you're
# totally screwed. Slackware updates versions of packaged
# software so including too much version prefix is a good way to
# have your install commands suddenly begin not installing
# anything.
slackpkg install openssh-7 git-2 </dev/null
- "checkout"
- run:
name: "Bootstrap test environment"
working_directory: "/tmp"
command: |
# Avoid the /nonexistent home directory in nobody's /etc/passwd
# entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try
# to test this checkout.
mv /root/project /tmp/project
# Python build/install toolchain wants to write to the source
# checkout, too.
chown --recursive nobody:nobody /tmp/project
slackpkg install \
ca-certificates \
sudo-1 \
make-4 \
automake-1 \
kernel-headers \
glibc-2 \
binutils-2 \
gcc-5 \
gcc-g++-5 \
python-2 \
libffi-3 \
libyaml-0 \
sqlite-3 \
icu4c-56 \
libmpc-1 </dev/null
slackpkg upgrade \
openssl-1 </dev/null
# neither virtualenv nor pip is packaged.
# do it the hard way.
# and it is extra hard since it is slackware.
slackpkg install \
cyrus-sasl-2 \
curl-7 </dev/null
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py
pip install virtualenv
- restore_cache: *RESTORE_HTTP_CACHE
- restore_cache: *RESTORE_WHEELHOUSE
- run: *SETUP_VIRTUALENV
- save_cache: *SAVE_HTTP_CACHE
- save_cache: *SAVE_WHEELHOUSE
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
build-image: &BUILD_IMAGE
# This is a template for a job to build a Docker image that has as much of
# the setup as we can manage already done and baked in. This cuts down on
# the per-job setup time the actual testing jobs have to perform - by
# perhaps 10% - 20%.
#
# https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
docker:
- image: "docker:17.05.0-ce-git"
environment:
DISTRO: "tahoelafsci/<DISTRO>:foo"
TAG: "tahoelafsci/distro:<TAG>"
steps:
- "checkout"
- "setup_remote_docker"
- run:
name: "Get openssl"
command: |
apk add --no-cache openssl
- run:
name: "Get Dockerhub secrets"
command: |
# If you create an encryption key like this:
#
# openssl enc -aes-256-cbc -k secret -P -md sha256
# From the output that looks like:
#
# salt=...
# key=...
# iv =...
#
# extract just the value for ``key``.
# then you can re-generate ``secret-env-cipher`` locally using the
# command:
#
# openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
#
# Make sure the key is set as the KEY environment variable in the
# CircleCI web interface. You can do this by visiting
# <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
# after logging in to CircleCI with an account in the tahoe-lafs
# CircleCI team.
#
# Then you can recover the environment plaintext (for example, to
# change and re-encrypt it) like just like CircleCI recovers it
# here:
#
openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
- run:
name: "Log in to Dockerhub"
command: |
. ~/.env
# TAHOELAFSCI_PASSWORD come from the secret env.
docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
- run:
name: "Build image"
command: |
docker \
build \
--build-arg TAG=${TAG} \
-t tahoelafsci/${DISTRO}:${TAG} \
-f ~/project/.circleci/Dockerfile.${DISTRO} \
~/project/
- run:
name: "Push image"
command: |
docker push tahoelafsci/${DISTRO}:${TAG}
build-image-debian-8:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "8"
build-image-debian-9:
<<: *BUILD_IMAGE
environment:
DISTRO: "debian"
TAG: "9"
build-image-ubuntu-16.04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "16.04"
build-image-ubuntu-18.04:
<<: *BUILD_IMAGE
environment:
DISTRO: "ubuntu"
TAG: "18.04"
build-image-centos-7:
<<: *BUILD_IMAGE
environment:
DISTRO: "centos"
TAG: "7"
build-image-fedora-28:
<<: *BUILD_IMAGE
environment:
DISTRO: "fedora"
TAG: "28"
build-image-fedora-29:
<<: *BUILD_IMAGE
environment:
DISTRO: "fedora"
TAG: "29"
build-image-slackware-14.2:
<<: *BUILD_IMAGE
environment:
DISTRO: "slackware"
TAG: "14.2"

37
.circleci/create-virtualenv.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies.
WHEELHOUSE_PATH="$1"
shift
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# Set up the virtualenv as a non-root user so we can run the test suite as a
# non-root user. See below.
virtualenv --python python2.7 "${BOOTSTRAP_VENV}"
# For convenience.
PIP="${BOOTSTRAP_VENV}/bin/pip"
# Tell pip where it can find any existing wheels.
export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem. Make sure this step comes before any other attempts to
# install things using pip!
"${PIP}" install certifi
# Get a new, awesome version of pip and setuptools. For example, the
# distro-packaged virtualenv's pip may not know about wheels.
"${PIP}" install --upgrade pip setuptools wheel

34
.circleci/fix-permissions.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies.
WHEELHOUSE_PATH="$1"
shift
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# The filesystem location of the root of the project source. We need this to
# know what wheels to get/build, of course.
PROJECT_ROOT="$1"
shift
# Most stuff is going to run as nobody. Here's a helper to make sure nobody
# can access necessary files.
CHOWN_NOBODY="chown --recursive nobody:$(id --group nobody)"
# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
usermod --home /tmp/nobody nobody
# Grant read access to nobody, the user which will eventually try to test this
# checkout.
${CHOWN_NOBODY} "${PROJECT_ROOT}"
# Create a place for some wheels to live.
mkdir -p "${WHEELHOUSE_PATH}"
${CHOWN_NOBODY} "${WHEELHOUSE_PATH}"

54
.circleci/populate-wheelhouse.sh Executable file
View File

@ -0,0 +1,54 @@
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# Basic Python packages that you just need to have around to do anything,
# practically speaking.
BASIC_DEPS="pip wheel"
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
REPORTING_DEPS="python-subunit junitxml subunitreporter"
# The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies.
WHEELHOUSE_PATH="$1"
shift
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# The filesystem location of the root of the project source. We need this to
# know what wheels to get/build, of course.
PROJECT_ROOT="$1"
shift
# For convenience.
PIP="${BOOTSTRAP_VENV}/bin/pip"
# Tell pip where it can find any existing wheels.
export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
# Populate the wheelhouse, if necessary.
"${PIP}" \
wheel \
--wheel-dir "${WHEELHOUSE_PATH}" \
"${PROJECT_ROOT}"[test,tor,i2p] \
${BASIC_DEPS} \
${TEST_DEPS} \
${REPORTING_DEPS}
# Not strictly wheelhouse population but ... Note we omit basic deps here.
# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
# have to ask.
"${PIP}" \
install \
${TEST_DEPS} \
${REPORTING_DEPS}

23
.circleci/prepare-image.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies.
WHEELHOUSE_PATH="$1"
shift
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# The filesystem location of the root of the project source. We need this to
# know what wheels to get/build, of course.
PROJECT_ROOT="$1"
shift
"${PROJECT_ROOT}"/.circleci/fix-permissions.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"
sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}"
sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/populate-wheelhouse.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"

36
.circleci/run-tests.sh
View File

@ -1,4 +1,17 @@
#!/bin/bash -e
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# The filesystem location of the root of the project source. We need this to
# know what wheels to get/build, of course.
PROJECT_ROOT="$1"
shift
ARTIFACTS=$1
shift
@ -15,13 +28,16 @@ if [ -n "${ARTIFACTS}" ]; then
# case where we expect not to end up here.
# Make sure we can actually write things to this directory.
sudo --user nobody mkdir -p "${ARTIFACTS}"
mkdir -p "${ARTIFACTS}"
SUBUNIT2="${ARTIFACTS}"/results.subunit2
# Use an intermediate directory here because CircleCI extracts some label
# information from its name.
JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
else
SUBUNIT2=""
JUNITXML=""
fi
# Run the test suite as a non-root user. This is the expected usage some
@ -34,14 +50,12 @@ fi
#
# Send the output directly to a file because transporting the binary subunit2
# via tox and then scraping it out is hideous and failure prone.
sudo \
SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" \
PIP_NO_INDEX="1" \
--set-home \
--user nobody \
/tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors"
export PIP_NO_INDEX="1"
${BOOTSTRAP_VENV}/bin/tox \
-c ${PROJECT_ROOT}/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS}
@ -49,5 +63,5 @@ sudo \
if [ -n "${ARTIFACTS}" ]; then
# Create a junitxml results area.
mkdir -p "$(dirname "${JUNITXML}")"
/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
fi

1
.circleci/secret-env-cipher Normal file
View File

@ -0,0 +1 @@
Salted__ •GPÁøÊ)|!÷[©U[‡ûvSÚ,F¿–m:ö š~ÓY[Uú_¸Fx×’¤Ÿ%<25>“4l×Ö»Š8¼œ¹„1öø‰/lƒÌ`nÆ^·Z]óqš¬æ¢&ø°÷£Ý‚‚ß%T¡n

77
.circleci/setup-virtualenv.sh
View File

@ -1,4 +1,22 @@
#!/bin/bash -e
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
# The filesystem location of the root of a virtualenv we can use to get/build
# wheels.
BOOTSTRAP_VENV="$1"
shift
# The filesystem location of the root of the project source. We need this to
# know what wheels to get/build, of course.
PROJECT_ROOT="$1"
shift
# The filesystem location of the wheelhouse which we'll populate with wheels
# for all of our dependencies.
WHEELHOUSE_PATH="$1"
shift
TAHOE_LAFS_TOX_ENVIRONMENT=$1
shift
@ -6,57 +24,18 @@ shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
# Tell pip where it can find any existing wheels.
export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
REPORTING_DEPS="python-subunit junitxml subunitreporter"
# Make sure the ownership of the pip cache directory is correct. The CircleCI
# cache management operations seem to mess it up. The cache directory might
# not exist if there was no matching cache to restore.
[ -e /tmp/nobody/.cache ] && chown --recursive nobody /tmp/nobody/.cache
# Set up the virtualenv as a non-root user so we can run the test suite as a
# non-root user. See below.
sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem. Make sure this step comes before any other attempts to
# install things using pip!
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install certifi
# Get a new, awesome version of pip and setuptools. For example, the
# distro-packaged virtualenv's pip may not know about wheels.
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install --upgrade pip setuptools wheel
# Populate the wheelhouse, if necessary.
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip \
wheel \
--wheel-dir "${WHEELHOUSE_PATH}" \
/tmp/project ${TEST_DEPS} ${REPORTING_DEPS}
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
# between the time dependencies change and the images are re-built and (b) the
# upcoming-deprecations job wants to install some dependencies from github and
# it's awkward to get that done any earlier than the tox run. So, we don't
# set it.
# Get everything else installed in it, too.
sudo --set-home -u nobody \
PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
/tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
"${BOOTSTRAP_VENV}"/bin/tox \
-c "${PROJECT_ROOT}"/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
--notest \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \

4
.gitignore vendored
View File

@ -44,3 +44,7 @@ zope.interface-*.egg
/coverage.xml
/smoke_magicfolder/
/.hypothesis/
# This is the plaintext of the private environment needed for some CircleCI
# operations. It's never supposed to be checked in.
secret-env-plain

25
integration/install-tor.sh
View File

@ -1,13 +1,22 @@
#!/bin/bash
# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
set -euxo pipefail
CODENAME=$(lsb_release --short --codename)
if [ "$(id -u)" != "0" ]; then
SUDO="sudo"
else
SUDO=""
fi
# Script to install Tor
echo "deb http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list
echo "deb-src http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list
set -ex
echo "deb http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
# Install Tor repo signing key
sudo apt-key add - <<EOF
# # Install Tor repo signing key
${SUDO} apt-key add - <<EOF
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQENBEqg7GsBCACsef8koRT8UyZxiv1Irke5nVpte54TDtTl1za1tOKfthmHbs2I
@ -707,5 +716,5 @@ I4Tesw==
-----END PGP PUBLIC KEY BLOCK-----
EOF
sudo apt-get --quiet update
sudo apt-get --quiet --yes install tor deb.torproject.org-keyring
${SUDO} apt-get --quiet update
${SUDO} apt-get --quiet --yes install tor deb.torproject.org-keyring

1
newsfragments/3024.other Normal file
View File

@ -0,0 +1 @@
CircleCI jobs are now faster as a result of pre-building configured Docker images for the CI jobs.