Merge pull request #595 from tahoe-lafs/3024.faster-circleci-builds

Faster CircleCI builds

Fixes: ticket:3024

.circleci/Dockerfile.centos

@@ -0,0 +1,26 @@
+ARG TAG
+FROM centos:${TAG}
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
+RUN yum install --assumeyes \
+    git \
+    sudo \
+    make automake gcc gcc-c++ \
+    python \
+    python-devel \
+    libffi-devel \
+    openssl-devel \
+    libyaml-devel \
+    /usr/bin/virtualenv \
+    net-tools
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

.circleci/Dockerfile.debian

@@ -0,0 +1,31 @@
+ARG TAG
+FROM debian:${TAG}
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+RUN apt-get --quiet update && \
+    apt-get --quiet --yes install \
+    	git \
+	lsb-release \
+        sudo \
+    	build-essential \
+	python2.7 \
+    	python2.7-dev \
+    	libffi-dev \
+    	libssl-dev \
+    	libyaml-dev \
+    	virtualenv
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+
+# Only the integration tests currently need this but it doesn't hurt to always
+# have it present and it's simpler than building a whole extra image just for
+# the integration tests.
+RUN ${BUILD_SRC_ROOT}/integration/install-tor.sh

.circleci/Dockerfile.fedora

@@ -0,0 +1,26 @@
+ARG TAG
+FROM fedora:${TAG}
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+# XXX net-tools is actually a Tahoe-LAFS runtime dependency!
+RUN yum install --assumeyes \
+    git \
+    sudo \
+    make automake gcc gcc-c++ \
+    python \
+    python-devel \
+    libffi-devel \
+    openssl-devel \
+    libyaml-devel \
+    /usr/bin/virtualenv \
+    net-tools
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

.circleci/Dockerfile.slackware

@@ -0,0 +1,49 @@
+ARG TAG
+FROM vbatts/slackware:${TAG}
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+# Be careful with slackpkg.  If the package name given doesn't match anything,
+# slackpkg still claims to succeed but you're totally screwed.  Slackware
+# updates versions of packaged software so including too much version prefix
+# is a good way to have your install commands suddenly begin not installing
+# anything.
+RUN slackpkg update && \
+    slackpkg install \
+        openssh-7 git-2 \
+        ca-certificates \
+        sudo-1 \
+        make-4 \
+        automake-1 \
+        kernel-headers \
+        glibc-2 \
+        binutils-2 \
+        gcc-5 \
+        gcc-g++-5 \
+        python-2 \
+        libffi-3 \
+        libyaml-0 \
+        sqlite-3 \
+        icu4c-56 \
+        libmpc-1 </dev/null && \
+    slackpkg upgrade \
+        openssl-1 </dev/null
+
+# neither virtualenv nor pip is packaged.
+# do it the hard way.
+# and it is extra hard since it is slackware.
+RUN slackpkg install \
+        cyrus-sasl-2 \
+        curl-7 </dev/null && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py && \
+    pip install virtualenv
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

.circleci/Dockerfile.ubuntu

@@ -0,0 +1,29 @@
+ARG TAG
+FROM ubuntu:${TAG}
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+# language-pack-en included to support the en_US LANG setting.
+# iproute2 necessary for automatic address detection/assignment.
+RUN apt-get --quiet update && \
+    apt-get --quiet --yes install git && \
+    apt-get --quiet --yes install \
+        sudo \
+        build-essential \
+        python2.7 \
+        python2.7-dev \
+        libffi-dev \
+        libssl-dev \
+        libyaml-dev \
+        virtualenv \
+        language-pack-en \
+        iproute2
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"

.circleci/bootstrap-test-environment.sh

@@ -1,27 +0,0 @@
-#!/bin/bash -e
-
-PROJECT=$1
-shift
-
-EXTRA_PACKAGES=$1
-shift
-
-# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
-usermod --home /tmp/nobody nobody
-
-# Grant read access to nobody, the user which will eventually try to test this
-# checkout.
-mv "${PROJECT}" /tmp/project
-
-# Python build/install toolchain wants to write to the source checkout, too.
-chown --recursive nobody:nogroup /tmp/project
-
-apt-get --quiet --yes install \
-    sudo \
-    build-essential \
-    python2.7 \
-    python2.7-dev \
-    libffi-dev \
-    libssl-dev \
-    libyaml-dev \
-    ${EXTRA_PACKAGES}

.circleci/config.yml

@@ -38,6 +38,28 @@ workflows:
             # integration tests.
             - "debian-9"
 
+  images:
+    # Build the Docker images used by the ci jobs.  This makes the ci jobs
+    # faster and takes various spurious failures out of the critical path.
+    triggers:
+      # Build once a day
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - "master"
+
+    jobs:
+      - "build-image-debian-8"
+      - "build-image-debian-9"
+      - "build-image-ubuntu-16.04"
+      - "build-image-ubuntu-18.04"
+      - "build-image-fedora-28"
+      - "build-image-fedora-29"
+      - "build-image-centos-7"
+      - "build-image-slackware-14.2"
+
 
 jobs:
   lint:
@@ -59,7 +81,8 @@ jobs:
 
   debian-9: &DEBIAN
     docker:
-      - image: "debian:9"
+      - image: "tahoelafsci/debian:9"
+        user: "nobody"
 
     environment: &UTF_8_ENVIRONMENT
       # Tell Hypothesis which configuration we want it to use.
@@ -67,8 +90,6 @@ jobs:
       # Tell the C runtime things about character encoding (mainly to do with
       # filenames and argv).
       LANG: "en_US.UTF-8"
-      # The package name for this software varies somewhat across distros.
-      EXTRA_PACKAGES: "virtualenv"
       # Select a tox environment to run for this job.
       TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
       # Additional arguments to pass to tox.
@@ -80,82 +101,28 @@ jobs:
       WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
       PIP_FIND_LINKS: "file:///tmp/wheelhouse"
 
+    # pip cannot install packages if the working directory is not readable.
+    # We want to run a lot of steps as nobody instead of as root.
+    working_directory: "/tmp/project"
+
     steps:
-      - run: &INSTALL_GIT
-          name: "Install Git"
-          command: |
-            apt-get --quiet update
-            apt-get --quiet --yes install git
-
       - "checkout"
-
-      - run: &BOOTSTRAP_TEST_ENVIRONMENT
-          name: "Bootstrap test environment"
-          command: |
-            ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
-
-      - restore_cache: &RESTORE_HTTP_CACHE
-          name: "Restoring pip HTTP cache"
-          keys:
-            # An exact match on the http cache key is great.  It should have
-            # exactly the packages (tgz, whl, whatever) we need.
-            - v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
-            # A prefix match is okay too.  It might have a
-            # partially-overlapping set of packages.  That's a head-start, at
-            # least.  We might have to download a few more things but at least
-            # we saved a little time.  After we download some more stuff we'll
-            # create a new cache entry with the full key above and the next
-            # build will get a better cache hit.
-            - v5-pip-http-
-
-      - restore_cache: &RESTORE_WHEELHOUSE
-          name: "Restoring wheelhouse"
-          keys:
-            # As above, an exact match is great.  Here, we also need to
-            # include the job name to make sure the platform ABI matches.
-            # There are binary wheels in this wheelhouse and we're not taking
-            # care to make manylinux1 wheels.  The binary wheels in this cache
-            # will only work on some Linux distros.
-            - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
-            # A partial match is okay too.  It'll get us at least some of the
-            # wheels.  We do need to keep the job name as part of the key or
-            # we might get binary wheels build against an incompatible ABI and
-            # we won't be able to use them (and they'll break the build rather
-            # than being ignored).
-            - v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}
-
       - run: &SETUP_VIRTUALENV
           name: "Setup virtualenv"
-          # pip cannot install packages if the working directory is not
-          # readable.
-          working_directory: "/tmp"
           command: |
             /tmp/project/.circleci/setup-virtualenv.sh \
+                "/tmp/venv" \
+                "/tmp/project" \
+                "${WHEELHOUSE_PATH}" \
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                 "${TAHOE_LAFS_TOX_ARGS}"
 
-      - save_cache: &SAVE_HTTP_CACHE
-          name: "Saving pip HTTP cache"
-          key: v5-pip-http-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
-          paths:
-            # Perfectly valid for Linux.  Note we exclude the wheel cache
-            # because we want this cache to be valid across all platforms and
-            # the wheels in the pip wheel cache are not necessarily so.
-            - "/tmp/nobody/.cache/pip/http"
-
-      - save_cache: &SAVE_WHEELHOUSE
-          name: "Caching wheelhouse"
-          key: v4-wheelhouse-{{ .Environment.CIRCLE_JOB }}-{{ checksum "/tmp/project/setup.py" }}-{{ checksum "/tmp/project/src/allmydata/_auto_deps.py" }}
-          paths:
-            - *WHEELHOUSE_PATH
-
       - run: &RUN_TESTS
           name: "Run test suite"
-          # Something about when it re-uses an existing environment blows up
-          # if the working directory is not readable.
-          working_directory: "/tmp"
           command: |
             /tmp/project/.circleci/run-tests.sh \
+                "/tmp/venv" \
+                "/tmp/project" \
                 "${ARTIFACTS_OUTPUT_PATH}" \
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                 "${TAHOE_LAFS_TOX_ARGS}"
@@ -179,15 +146,15 @@ jobs:
 
       - run: &SUBMIT_COVERAGE
           name: "Submit coverage results"
-          working_directory: "/tmp/project"
           command: |
-            /tmp/tests/bin/codecov
+            /tmp/venv/bin/codecov
 
 
   debian-8:
     <<: *DEBIAN
     docker:
-      - image: "debian:8"
+      - image: "tahoelafsci/debian:8"
+        user: "nobody"
 
 
   c-locale:
@@ -228,101 +195,42 @@ jobs:
       # Disable artifact collection because py.test can't produce any.
       ARTIFACTS_OUTPUT_PATH: ""
 
-
     steps:
-      # DRY, YAML-style.  See the debian-9 steps.
-      - run: *INSTALL_GIT
       - "checkout"
-      - run: *BOOTSTRAP_TEST_ENVIRONMENT
+      # DRY, YAML-style.  See the debian-9 steps.
-      - restore_cache: *RESTORE_HTTP_CACHE
-      - restore_cache: *RESTORE_WHEELHOUSE
       - run: *SETUP_VIRTUALENV
-      - run:
-          name: "Install Tor"
-          command: |
-            # Don't forget that we moved the source tree to /tmp.
-            /tmp/project/integration/install-tor.sh
-      - save_cache: *SAVE_HTTP_CACHE
-      - save_cache: *SAVE_WHEELHOUSE
       - run: *RUN_TESTS
 
 
   ubuntu-16.04:
     <<: *DEBIAN
     docker:
-      - image: "ubuntu:16.04"
+      - image: "tahoelafsci/ubuntu:16.04"
-
+        user: "nobody"
-    environment:
-      <<: *UTF_8_ENVIRONMENT
-      # Necessary for en_US LANG setting.
-      EXTRA_PACKAGES: "virtualenv language-pack-en"
 
 
   ubuntu-18.04:
     <<: *DEBIAN
     docker:
-      - image: "ubuntu:18.04"
+      - image: "tahoelafsci/ubuntu:18.04"
-
+        user: "nobody"
-    environment:
-      <<: *UTF_8_ENVIRONMENT
-      # Necessary for automatic address detection/assignment.
-      EXTRA_PACKAGES: "virtualenv iproute2"
 
 
   centos-7: &RHEL_DERIV
     docker:
-      - image: "centos:7"
+      - image: "tahoelafsci/centos:7"
+        user: "nobody"
 
     environment: *UTF_8_ENVIRONMENT
 
+    # pip cannot install packages if the working directory is not readable.
+    # We want to run a lot of steps as nobody instead of as root.
+    working_directory: "/tmp/project"
+
     steps:
-      - run:
-          name: "Install Git"
-          command: |
-            yum install --assumeyes git
-
       - "checkout"
-
-      - run:
-          name: "Bootstrap test environment"
-          working_directory: "/tmp"
-          command: |
-            # Avoid the /nonexistent home directory in nobody's /etc/passwd
-            # entry.
-            usermod --home /tmp/nobody nobody
-
-            # Grant read access to nobody, the user which will eventually try
-            # to test this checkout.
-            mv /root/project /tmp/project
-
-            # Python build/install toolchain wants to write to the source
-            # checkout, too.
-            chown --recursive nobody:nobody /tmp/project
-
-            yum install --assumeyes \
-                sudo \
-                make automake gcc gcc-c++ \
-                python \
-                python-devel \
-                libffi-devel \
-                openssl-devel \
-                libyaml-devel
-            yum install --assumeyes /usr/bin/virtualenv
-
-            # XXX net-tools is actually a Tahoe-LAFS runtime dependency!
-            yum install --assumeyes \
-                net-tools
-
-      - restore_cache: *RESTORE_HTTP_CACHE
-      - restore_cache: *RESTORE_WHEELHOUSE
-
       - run: *SETUP_VIRTUALENV
-
-      - save_cache: *SAVE_HTTP_CACHE
-      - save_cache: *SAVE_WHEELHOUSE
-
       - run: *RUN_TESTS
-
       - store_test_results: *STORE_TEST_RESULTS
       - store_artifacts: *STORE_TEST_LOG
       - store_artifacts: *STORE_OTHER_ARTIFACTS
@@ -332,94 +240,169 @@ jobs:
   fedora-28:
     <<: *RHEL_DERIV
     docker:
-      - image: "fedora:28"
+      - image: "tahoelafsci/fedora:28"
+        user: "nobody"
 
 
   fedora-29:
     <<: *RHEL_DERIV
     docker:
-      - image: "fedora:29"
+      - image: "tahoelafsci/fedora:29"
+        user: "nobody"
 
 
   slackware-14.2:
     docker:
-      - image: "vbatts/slackware:14.2"
+      - image: "tahoelafsci/slackware:14.2"
+        user: "nobody"
 
     environment: *UTF_8_ENVIRONMENT
 
+    # pip cannot install packages if the working directory is not readable.
+    # We want to run a lot of steps as nobody instead of as root.
+    working_directory: "/tmp/project"
+
     steps:
-      - run:
-          name: "Install Git"
-          command: |
-            slackpkg update
-
-            # Be careful with slackpkg.  If the package name given doesn't
-            # match anything, slackpkg still claims to succeed but you're
-            # totally screwed.  Slackware updates versions of packaged
-            # software so including too much version prefix is a good way to
-            # have your install commands suddenly begin not installing
-            # anything.
-            slackpkg install openssh-7 git-2 </dev/null
-
       - "checkout"
-
-      - run:
-          name: "Bootstrap test environment"
-          working_directory: "/tmp"
-          command: |
-            # Avoid the /nonexistent home directory in nobody's /etc/passwd
-            # entry.
-            usermod --home /tmp/nobody nobody
-
-            # Grant read access to nobody, the user which will eventually try
-            # to test this checkout.
-            mv /root/project /tmp/project
-
-            # Python build/install toolchain wants to write to the source
-            # checkout, too.
-            chown --recursive nobody:nobody /tmp/project
-
-            slackpkg install \
-                ca-certificates \
-                sudo-1 \
-                make-4 \
-                automake-1 \
-                kernel-headers \
-                glibc-2 \
-                binutils-2 \
-                gcc-5 \
-                gcc-g++-5 \
-                python-2 \
-                libffi-3 \
-                libyaml-0 \
-                sqlite-3 \
-                icu4c-56 \
-                libmpc-1 </dev/null
-
-            slackpkg upgrade \
-                openssl-1 </dev/null
-
-            # neither virtualenv nor pip is packaged.
-            # do it the hard way.
-            # and it is extra hard since it is slackware.
-            slackpkg install \
-                cyrus-sasl-2 \
-                curl-7 </dev/null
-            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-            python get-pip.py
-            pip install virtualenv
-
-      - restore_cache: *RESTORE_HTTP_CACHE
-      - restore_cache: *RESTORE_WHEELHOUSE
-
       - run: *SETUP_VIRTUALENV
-
-      - save_cache: *SAVE_HTTP_CACHE
-      - save_cache: *SAVE_WHEELHOUSE
-
       - run: *RUN_TESTS
-
       - store_test_results: *STORE_TEST_RESULTS
       - store_artifacts: *STORE_TEST_LOG
       - store_artifacts: *STORE_OTHER_ARTIFACTS
       - run: *SUBMIT_COVERAGE
+
+  build-image: &BUILD_IMAGE
+    # This is a template for a job to build a Docker image that has as much of
+    # the setup as we can manage already done and baked in.  This cuts down on
+    # the per-job setup time the actual testing jobs have to perform - by
+    # perhaps 10% - 20%.
+    #
+    # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
+    docker:
+      - image: "docker:17.05.0-ce-git"
+
+    environment:
+      DISTRO: "tahoelafsci/<DISTRO>:foo"
+      TAG: "tahoelafsci/distro:<TAG>"
+
+    steps:
+      - "checkout"
+      - "setup_remote_docker"
+      - run:
+          name: "Get openssl"
+          command: |
+            apk add --no-cache openssl
+      - run:
+          name: "Get Dockerhub secrets"
+          command: |
+            # If you create an encryption key like this:
+            #
+            #   openssl enc -aes-256-cbc -k secret -P -md sha256
+
+            # From the output that looks like:
+            #
+            #  salt=...
+            #  key=...
+            #  iv =...
+            #
+            # extract just the value for ``key``.
+
+            # then you can re-generate ``secret-env-cipher`` locally using the
+            # command:
+            #
+            #   openssl aes-256-cbc -e -md sha256 -in secret-env-plain -out .circleci/secret-env-cipher -pass env:KEY
+            #
+            # Make sure the key is set as the KEY environment variable in the
+            # CircleCI web interface.  You can do this by visiting
+            # <https://circleci.com/gh/tahoe-lafs/tahoe-lafs/edit#env-vars>
+            # after logging in to CircleCI with an account in the tahoe-lafs
+            # CircleCI team.
+            #
+            # Then you can recover the environment plaintext (for example, to
+            # change and re-encrypt it) like just like CircleCI recovers it
+            # here:
+            #
+            openssl aes-256-cbc -d -md sha256 -in .circleci/secret-env-cipher -pass env:KEY >> ~/.env
+      - run:
+          name: "Log in to Dockerhub"
+          command: |
+            . ~/.env
+            # TAHOELAFSCI_PASSWORD come from the secret env.
+            docker login -u tahoelafsci -p ${TAHOELAFSCI_PASSWORD}
+      - run:
+          name: "Build image"
+          command: |
+            docker \
+                build \
+                --build-arg TAG=${TAG} \
+                -t tahoelafsci/${DISTRO}:${TAG} \
+                -f ~/project/.circleci/Dockerfile.${DISTRO} \
+                ~/project/
+      - run:
+          name: "Push image"
+          command: |
+            docker push tahoelafsci/${DISTRO}:${TAG}
+
+
+  build-image-debian-8:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "debian"
+      TAG: "8"
+
+
+  build-image-debian-9:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "debian"
+      TAG: "9"
+
+
+  build-image-ubuntu-16.04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "16.04"
+
+
+  build-image-ubuntu-18.04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "18.04"
+
+
+  build-image-centos-7:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "centos"
+      TAG: "7"
+
+
+  build-image-fedora-28:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "fedora"
+      TAG: "28"
+
+
+  build-image-fedora-29:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "fedora"
+      TAG: "29"
+
+
+  build-image-slackware-14.2:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "slackware"
+      TAG: "14.2"

.circleci/create-virtualenv.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# The filesystem location of the wheelhouse which we'll populate with wheels
+# for all of our dependencies.
+WHEELHOUSE_PATH="$1"
+shift
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# Set up the virtualenv as a non-root user so we can run the test suite as a
+# non-root user.  See below.
+virtualenv --python python2.7 "${BOOTSTRAP_VENV}"
+
+# For convenience.
+PIP="${BOOTSTRAP_VENV}/bin/pip"
+
+# Tell pip where it can find any existing wheels.
+export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
+
+# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
+# a package to be installed (with setuptools) then it'll fail on certain
+# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
+# requirements (TLS >= 1.2) are incompatible with the old TLS clients
+# available to those systems.  Installing it ahead of time (with pip) avoids
+# this problem.  Make sure this step comes before any other attempts to
+# install things using pip!
+"${PIP}" install certifi
+
+# Get a new, awesome version of pip and setuptools.  For example, the
+# distro-packaged virtualenv's pip may not know about wheels.
+"${PIP}" install --upgrade pip setuptools wheel

.circleci/fix-permissions.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# The filesystem location of the wheelhouse which we'll populate with wheels
+# for all of our dependencies.
+WHEELHOUSE_PATH="$1"
+shift
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# The filesystem location of the root of the project source.  We need this to
+# know what wheels to get/build, of course.
+PROJECT_ROOT="$1"
+shift
+
+# Most stuff is going to run as nobody.  Here's a helper to make sure nobody
+# can access necessary files.
+CHOWN_NOBODY="chown --recursive nobody:$(id --group nobody)"
+
+# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
+usermod --home /tmp/nobody nobody
+
+# Grant read access to nobody, the user which will eventually try to test this
+# checkout.
+${CHOWN_NOBODY} "${PROJECT_ROOT}"
+
+# Create a place for some wheels to live.
+mkdir -p "${WHEELHOUSE_PATH}"
+${CHOWN_NOBODY} "${WHEELHOUSE_PATH}"

.circleci/populate-wheelhouse.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# Basic Python packages that you just need to have around to do anything,
+# practically speaking.
+BASIC_DEPS="pip wheel"
+
+# Python packages we need to support the test infrastructure.  *Not* packages
+# Tahoe-LAFS itself (implementation or test suite) need.
+TEST_DEPS="tox codecov"
+
+# Python packages we need to generate test reports for CI infrastructure.
+# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
+REPORTING_DEPS="python-subunit junitxml subunitreporter"
+
+# The filesystem location of the wheelhouse which we'll populate with wheels
+# for all of our dependencies.
+WHEELHOUSE_PATH="$1"
+shift
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# The filesystem location of the root of the project source.  We need this to
+# know what wheels to get/build, of course.
+PROJECT_ROOT="$1"
+shift
+
+# For convenience.
+PIP="${BOOTSTRAP_VENV}/bin/pip"
+
+# Tell pip where it can find any existing wheels.
+export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
+
+# Populate the wheelhouse, if necessary.
+"${PIP}" \
+    wheel \
+    --wheel-dir "${WHEELHOUSE_PATH}" \
+    "${PROJECT_ROOT}"[test,tor,i2p] \
+    ${BASIC_DEPS} \
+    ${TEST_DEPS} \
+    ${REPORTING_DEPS}
+
+# Not strictly wheelhouse population but ... Note we omit basic deps here.
+# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
+# have to ask.
+"${PIP}" \
+    install \
+    ${TEST_DEPS} \
+    ${REPORTING_DEPS}

.circleci/prepare-image.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# The filesystem location of the wheelhouse which we'll populate with wheels
+# for all of our dependencies.
+WHEELHOUSE_PATH="$1"
+shift
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# The filesystem location of the root of the project source.  We need this to
+# know what wheels to get/build, of course.
+PROJECT_ROOT="$1"
+shift
+
+"${PROJECT_ROOT}"/.circleci/fix-permissions.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"
+sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}"
+sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/populate-wheelhouse.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"

.circleci/run-tests.sh

@@ -1,4 +1,17 @@
-#!/bin/bash -e
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# The filesystem location of the root of the project source.  We need this to
+# know what wheels to get/build, of course.
+PROJECT_ROOT="$1"
+shift
 
 ARTIFACTS=$1
 shift
@@ -15,13 +28,16 @@ if [ -n "${ARTIFACTS}" ]; then
     # case where we expect not to end up here.
 
     # Make sure we can actually write things to this directory.
-    sudo --user nobody mkdir -p "${ARTIFACTS}"
+    mkdir -p "${ARTIFACTS}"
 
     SUBUNIT2="${ARTIFACTS}"/results.subunit2
 
     # Use an intermediate directory here because CircleCI extracts some label
     # information from its name.
     JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
+else
+    SUBUNIT2=""
+    JUNITXML=""
 fi
 
 # Run the test suite as a non-root user.  This is the expected usage some
@@ -34,14 +50,12 @@ fi
 #
 # Send the output directly to a file because transporting the binary subunit2
 # via tox and then scraping it out is hideous and failure prone.
-sudo \
+export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
-    SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
+export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors"
-    TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors" \
+export PIP_NO_INDEX="1"
-    PIP_NO_INDEX="1" \
+
-    --set-home \
+${BOOTSTRAP_VENV}/bin/tox \
-    --user nobody \
+    -c ${PROJECT_ROOT}/tox.ini \
-    /tmp/tests/bin/tox \
-    -c /tmp/project/tox.ini \
     --workdir /tmp/tahoe-lafs.tox \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
     ${TAHOE_LAFS_TOX_ARGS}
@@ -49,5 +63,5 @@ sudo \
 if [ -n "${ARTIFACTS}" ]; then
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    /tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
+    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
 fi

.circleci/secret-env-cipher

@@ -0,0 +1 @@
+Salted__ •GPÁøÊ)|!÷[©U[‡ûvSÚ,F¿–m:ö š~ÓY[Uú_¸Fx×’¤Ÿ%<25>“4l×Ö»Š8¼œ¹„1öø‰/lƒÌ`nÆ^·Z]óqš¬æ¢&ø°÷£Ý‚‚ß%T¡n

.circleci/setup-virtualenv.sh

@@ -1,4 +1,22 @@
-#!/bin/bash -e
+#!/bin/bash
+
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+# The filesystem location of the root of a virtualenv we can use to get/build
+# wheels.
+BOOTSTRAP_VENV="$1"
+shift
+
+# The filesystem location of the root of the project source.  We need this to
+# know what wheels to get/build, of course.
+PROJECT_ROOT="$1"
+shift
+
+# The filesystem location of the wheelhouse which we'll populate with wheels
+# for all of our dependencies.
+WHEELHOUSE_PATH="$1"
+shift
 
 TAHOE_LAFS_TOX_ENVIRONMENT=$1
 shift
@@ -6,57 +24,18 @@ shift
 TAHOE_LAFS_TOX_ARGS=$1
 shift || :
 
-# Python packages we need to support the test infrastructure.  *Not* packages
+# Tell pip where it can find any existing wheels.
-# Tahoe-LAFS itself (implementation or test suite) need.
+export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
-TEST_DEPS="tox codecov"
 
-# Python packages we need to generate test reports for CI infrastructure.
+# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
+# between the time dependencies change and the images are re-built and (b) the
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
+# upcoming-deprecations job wants to install some dependencies from github and
-
+# it's awkward to get that done any earlier than the tox run.  So, we don't
-# Make sure the ownership of the pip cache directory is correct.  The CircleCI
+# set it.
-# cache management operations seem to mess it up.  The cache directory might
-# not exist if there was no matching cache to restore.
-[ -e /tmp/nobody/.cache ] && chown --recursive nobody /tmp/nobody/.cache
-
-# Set up the virtualenv as a non-root user so we can run the test suite as a
-# non-root user.  See below.
-sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
-
-# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
-# a package to be installed (with setuptools) then it'll fail on certain
-# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
-# requirements (TLS >= 1.2) are incompatible with the old TLS clients
-# available to those systems.  Installing it ahead of time (with pip) avoids
-# this problem.  Make sure this step comes before any other attempts to
-# install things using pip!
-sudo --set-home -u nobody \
-     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
-     /tmp/tests/bin/pip install certifi
-
-# Get a new, awesome version of pip and setuptools.  For example, the
-# distro-packaged virtualenv's pip may not know about wheels.
-sudo --set-home -u nobody \
-     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
-     /tmp/tests/bin/pip install --upgrade pip setuptools wheel
-
-# Populate the wheelhouse, if necessary.
-sudo --set-home -u nobody \
-     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
-     /tmp/tests/bin/pip \
-     wheel \
-     --wheel-dir "${WHEELHOUSE_PATH}" \
-     /tmp/project ${TEST_DEPS} ${REPORTING_DEPS}
-
-sudo --set-home -u nobody \
-     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
-     /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
 
 # Get everything else installed in it, too.
-sudo --set-home -u nobody \
+"${BOOTSTRAP_VENV}"/bin/tox \
-     PIP_FIND_LINKS="${PIP_FIND_LINKS}" \
+     -c "${PROJECT_ROOT}"/tox.ini \
-     /tmp/tests/bin/tox \
-     -c /tmp/project/tox.ini \
      --workdir /tmp/tahoe-lafs.tox \
      --notest \
      -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \

.gitignore

@@ -44,3 +44,7 @@ zope.interface-*.egg
 /coverage.xml
 /smoke_magicfolder/
 /.hypothesis/
+
+# This is the plaintext of the private environment needed for some CircleCI
+# operations.  It's never supposed to be checked in.
+secret-env-plain

integration/install-tor.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
 
+# https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
+set -euxo pipefail
+
+CODENAME=$(lsb_release --short --codename)
+
+if [ "$(id -u)" != "0" ]; then
+    SUDO="sudo"
+else
+    SUDO=""
+fi
+
 # Script to install Tor
+echo "deb http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list
+echo "deb-src http://deb.torproject.org/torproject.org ${CODENAME} main" | ${SUDO} tee -a /etc/apt/sources.list
 
-set -ex
+# # Install Tor repo signing key
-echo "deb http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
+${SUDO} apt-key add - <<EOF
-echo "deb-src http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
-
-# Install Tor repo signing key
-sudo apt-key add - <<EOF
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
 mQENBEqg7GsBCACsef8koRT8UyZxiv1Irke5nVpte54TDtTl1za1tOKfthmHbs2I
@@ -707,5 +716,5 @@ I4Tesw==
 -----END PGP PUBLIC KEY BLOCK-----
 EOF
 
-sudo apt-get --quiet update
+${SUDO} apt-get --quiet update
-sudo apt-get --quiet --yes install tor deb.torproject.org-keyring
+${SUDO} apt-get --quiet --yes install tor deb.torproject.org-keyring

newsfragments/3024.other

@@ -0,0 +1 @@
+CircleCI jobs are now faster as a result of pre-building configured Docker images for the CI jobs.