Merge pull request #626 from tahoe-lafs/2479.pypy-ci-again

Test against PyPy on CI

Fixes: ticket:2479

.circleci/Dockerfile.centos

@@ -23,4 +23,4 @@ RUN yum install --assumeyes \
 # *update* this checkout on each job run, saving us more time per-job.
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.debian

@@ -23,7 +23,7 @@ RUN apt-get --quiet update && \
 # *update* this checkout on each job run, saving us more time per-job.
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"
 
 # Only the integration tests currently need this but it doesn't hurt to always
 # have it present and it's simpler than building a whole extra image just for

.circleci/Dockerfile.fedora

@@ -23,4 +23,4 @@ RUN yum install --assumeyes \
 # *update* this checkout on each job run, saving us more time per-job.
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.pypy

@@ -0,0 +1,23 @@
+FROM pypy:2.7-7.1.1-jessie
+
+ENV WHEELHOUSE_PATH /tmp/wheelhouse
+ENV VIRTUALENV_PATH /tmp/venv
+# This will get updated by the CircleCI checkout step.
+ENV BUILD_SRC_ROOT /tmp/project
+
+RUN apt-get --quiet update && \
+    apt-get --quiet --yes install \
+        git \
+        lsb-release \
+        sudo \
+        build-essential \
+        libffi-dev \
+        libssl-dev \
+        libyaml-dev \
+        virtualenv
+
+# Get the project source.  This is better than it seems.  CircleCI will
+# *update* this checkout on each job run, saving us more time per-job.
+COPY . ${BUILD_SRC_ROOT}
+
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "pypy"

.circleci/Dockerfile.slackware

@@ -46,4 +46,4 @@ RUN slackpkg install \
 # *update* this checkout on each job run, saving us more time per-job.
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/Dockerfile.ubuntu

@@ -26,4 +26,4 @@ RUN apt-get --quiet update && \
 # *update* this checkout on each job run, saving us more time per-job.
 COPY . ${BUILD_SRC_ROOT}
 
-RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}"
+RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7"

.circleci/config.yml

@@ -25,6 +25,9 @@ workflows:
 
       - "slackware-14.2"
 
+      # Test against PyPy 2.7/7.1.1
+      - "pypy2.7-7.1"
+
       # Other assorted tasks and configurations
       - "lint"
       - "deprecations"
@@ -59,6 +62,7 @@ workflows:
       - "build-image-fedora-29"
       - "build-image-centos-7"
       - "build-image-slackware-14.2"
+      - "build-image-pypy-2.7-7.1.1-jessie"
 
 
 jobs:
@@ -85,13 +89,17 @@ jobs:
         user: "nobody"
 
     environment: &UTF_8_ENVIRONMENT
+      # In general, the test suite is not allowed to fail while the job
+      # succeeds.  But you can set this to "yes" if you want it to be
+      # otherwise.
+      ALLOWED_FAILURE: "no"
       # Tell Hypothesis which configuration we want it to use.
       TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
       # Tell the C runtime things about character encoding (mainly to do with
       # filenames and argv).
       LANG: "en_US.UTF-8"
       # Select a tox environment to run for this job.
-      TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
       # The path in which test artifacts will be placed.
@@ -123,6 +131,7 @@ jobs:
             /tmp/project/.circleci/run-tests.sh \
                 "/tmp/venv" \
                 "/tmp/project" \
+                "${ALLOWED_FAILURE}" \
                 "${ARTIFACTS_OUTPUT_PATH}" \
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                 "${TAHOE_LAFS_TOX_ARGS}"
@@ -157,6 +166,18 @@ jobs:
         user: "nobody"
 
 
+  pypy2.7-7.1:
+    <<: *DEBIAN
+    docker:
+      - image: "tahoelafsci/pypy:2.7-7.1.1-jessie"
+        user: "nobody"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage"
+      ALLOWED_FAILURE: "yes"
+
+
   c-locale:
     <<: *DEBIAN
 
@@ -406,3 +427,11 @@ jobs:
     environment:
       DISTRO: "slackware"
       TAG: "14.2"
+
+
+  build-image-pypy-2.7-7.1.1-jessie:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "pypy"
+      TAG: "2.7-7.1.1-jessie"

.circleci/create-virtualenv.sh

@@ -13,9 +13,14 @@ shift
 BOOTSTRAP_VENV="$1"
 shift
 
+# The basename of the Python executable (found on PATH) that will be used with
+# this image.  This lets us create a virtualenv that uses the correct Python.
+PYTHON="$1"
+shift
+
 # Set up the virtualenv as a non-root user so we can run the test suite as a
 # non-root user.  See below.
-virtualenv --python python2.7 "${BOOTSTRAP_VENV}"
+virtualenv --python "${PYTHON}" "${BOOTSTRAP_VENV}"
 
 # For convenience.
 PIP="${BOOTSTRAP_VENV}/bin/pip"

.circleci/prepare-image.sh

@@ -18,6 +18,11 @@ shift
 PROJECT_ROOT="$1"
 shift
 
+# The basename of the Python executable (found on PATH) that will be used with
+# this image.  This lets us create a virtualenv that uses the correct Python.
+PYTHON="$1"
+shift
+
 "${PROJECT_ROOT}"/.circleci/fix-permissions.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"
-sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}"
+sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/create-virtualenv.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PYTHON}"
 sudo --set-home -u nobody "${PROJECT_ROOT}"/.circleci/populate-wheelhouse.sh "${WHEELHOUSE_PATH}" "${BOOTSTRAP_VENV}" "${PROJECT_ROOT}"

.circleci/run-tests.sh

@@ -13,6 +13,9 @@ shift
 PROJECT_ROOT="$1"
 shift
 
+ALLOWED_FAILURE="$1"
+shift
+
 ARTIFACTS=$1
 shift
 
@@ -54,14 +57,20 @@ export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
 export TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file --rterrors"
 export PIP_NO_INDEX="1"
 
+if [ "${ALLOWED_FAILURE}" = "yes" ]; then
+    alternative="true"
+else
+    alternative="false"
+fi
+
 ${BOOTSTRAP_VENV}/bin/tox \
     -c ${PROJECT_ROOT}/tox.ini \
     --workdir /tmp/tahoe-lafs.tox \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
-    ${TAHOE_LAFS_TOX_ARGS}
+    ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
 
 if [ -n "${ARTIFACTS}" ]; then
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
+    ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

newsfragments/2479.other

@@ -0,0 +1 @@
+Tahoe-LAFS now tests for PyPy compatibility on CI.

tox.ini

@@ -7,7 +7,7 @@
 twisted = 1
 
 [tox]
-envlist = py27
+envlist = {py27,pypy27}{-coverage,}
 minversion = 2.4
 
 [testenv]