From 00e3a04c7e2b52df838be84a5dc1b3efe699b81b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone <exarkun@twistedmatrix.com> Date: Fri, 15 Jun 2018 14:21:33 -0400 Subject: [PATCH] Factor steps out for use by machine executer --- .circleci/bootstrap-test-environment.sh | 22 +++++ .circleci/config.yml | 107 ++++++++++-------------- .circleci/install-git.sh | 4 + .circleci/run-build-locally.sh | 10 +++ .circleci/run-tests.sh | 10 +++ .circleci/setup-virtualenv.sh | 8 ++ 6 files changed, 99 insertions(+), 62 deletions(-) create mode 100755 .circleci/bootstrap-test-environment.sh create mode 100755 .circleci/install-git.sh create mode 100755 .circleci/run-build-locally.sh create mode 100755 .circleci/run-tests.sh create mode 100755 .circleci/setup-virtualenv.sh diff --git a/.circleci/bootstrap-test-environment.sh b/.circleci/bootstrap-test-environment.sh new file mode 100755 index 000000000..232bcb380 --- /dev/null +++ b/.circleci/bootstrap-test-environment.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +# Avoid the /nonexistent home directory in nobody's /etc/passwd entry. +usermod --home /tmp/nobody nobody + +# Grant read access to nobody, the user which will eventually try to test this +# checkout. +mv /root/project /tmp/project + +# Python build/install toolchain wants to write to the source checkout, too. +chown --recursive nobody:nogroup /tmp/project + +apt-get --quiet --yes install \ + sudo \ + build-essential \ + python2.7 \ + python2.7-dev \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + virtualenv \ + ${EXTRA_PACKAGES} diff --git a/.circleci/config.yml b/.circleci/config.yml index 6971c30a0..8de86961f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,17 +5,17 @@ workflows: version: 2 ci: jobs: - - "lint" - - "debian-8" - - "debian-9" - - "ubuntu-16.04" - - "ubuntu-18.04" - - "centos-7" - - "fedora-27" - - "fedora-28" + # - "lint" + # - "debian-8" + # - "debian-9" + # - "ubuntu-16.04" + # - "ubuntu-18.04" + # - "centos-7" + # - "fedora-27" + # - "fedora-28" - "magic-folder-ubuntu-14.04" - - "deprecations" + # - "deprecations" jobs: lint: @@ -51,80 +51,42 @@ jobs: TAHOE_LAFS_TOX_ARGS: "" steps: - - run: + - run: &INSTALL_GIT node: "Install Git" - command: | - apt-get --quiet update - apt-get --quiet --yes install git + command: "${CIRCLE_WORKING_DIRECTORY}/.circleci/install-git.sh" - "checkout" - - run: + - run: &BOOTSTRAP_TEST_ENVIRONMENT name: "Bootstrap test environment" - command: | - # Avoid the /nonexistent home directory in nobody's /etc/passwd - # entry. - usermod --home /tmp/nobody nobody - - # Grant read access to nobody, the user which will eventually try - # to test this checkout. - mv /root/project /tmp/project - - # Python build/install toolchain wants to write to the source - # checkout, too. - chown --recursive nobody:nogroup /tmp/project - - apt-get --quiet --yes install \ - sudo \ - build-essential \ - python2.7 \ - python2.7-dev \ - libffi-dev \ - libssl-dev \ - libyaml-dev \ - virtualenv \ - ${EXTRA_PACKAGES} + command: "${CIRCLE_WORKING_DIRECTORY}/.circleci/bootstrap-test-environment.sh" - run: &SETUP_VIRTUALENV name: "Setup virtualenv" # pip cannot install packages if the working directory is not # readable. working_directory: "/tmp" - command: | - # Set up the virtualenv as a non-root user so we can run the test - # suite as a non-root user. See below. - sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests - sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov - # Get everything installed in it, too. - sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS} + command: "${CIRCLE_WORKING_DIRECTORY}/.circleci/setup-virtualenv.sh" - run: &RUN_TESTS name: "Run test suite" # Something about when it re-uses an existing environment blows up # if the working directory is not readable. working_directory: "/tmp" - command: | - # Run the test suite as a non-root user. This is the expected - # usage some small areas of the test suite assume non-root - # privileges (such as unreadable files being unreadable). - # - # Also run with /tmp as a workdir because the non-root user won't - # be able to create the tox working filesystem state in the source - # checkout because it is owned by root. - sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e ${TAHOE_LAFS_TOX_ENVIRONMENT} + command: "${CIRCLE_WORKING_DIRECTORY}/.circleci/setup-virtualenv.sh" - - store_artifacts: &STORE_ARTIFACTS + - store_artifacts: &STORE_TEST_LOG # Despite passing --workdir /tmp to tox above, it still runs trial # in the project source checkout. path: "/tmp/project/_trial_temp/test.log" - - store_artifacts: + - store_artifacts: &STORE_OTHER_ARTIFACTS # Store any other artifacts, too. This is handy to allow other jobs # sharing most of the definition of this one to be able to # contribute artifacts easily. path: "/tmp/artifacts" - - run: + - run: &SUBMIT_COVERAGE name: "Submit coverage results" command: | /tmp/tests/bin/codecov @@ -218,7 +180,9 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - - store_artifacts: *STORE_ARTIFACTS + - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE fedora-27: @@ -234,11 +198,6 @@ jobs: magic-folder-ubuntu-14.04: - <<: *DEBIAN - - # Turn off the inherited docker executor. - docker: null - machine: enabled: true image: "circleci/classic:201711-01" @@ -246,3 +205,27 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ARGS: "-- allmydata.test.test_magic_folder" + + # Unfortunately, duplicate all the steps here but run with `sudo`. + steps: + - run: + <<: *INSTALL_GIT + command: "sudo ${CIRCLE_WORKING_DIRECTORY}/.circleci/install-git.sh" + + - "checkout" + + - run: + <<: *BOOTSTRAP_TEST_ENVIRONMENT + command: "${CIRCLE_WORKING_DIRECTORY}/.circleci/bootstrap-test-environment.sh" + + - run: + <<: *SETUP_VIRTUALENV + command: "sudo ${CIRCLE_WORKING_DIRECTORY}/.circleci/setup-virtualenv.sh" + + - run: + <<: *RUN_TESTS + command: "sudo ${CIRCLE_WORKING_DIRECTORY}/.circleci/setup-virtualenv.sh" + + - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE diff --git a/.circleci/install-git.sh b/.circleci/install-git.sh new file mode 100755 index 000000000..a9231e978 --- /dev/null +++ b/.circleci/install-git.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +apt-get --quiet update +apt-get --quiet --yes install git diff --git a/.circleci/run-build-locally.sh b/.circleci/run-build-locally.sh new file mode 100755 index 000000000..32b801277 --- /dev/null +++ b/.circleci/run-build-locally.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CIRCLE_TOKEN=efb53124be82dd4b3153bc0e3f60de71da629d59 + +curl --user ${CIRCLE_TOKEN}: \ + --request POST \ + --form revision=$(git rev-parse HEAD) \ + --form config=@config.yml \ + --form notify=false \ + https://circleci.com/api/v1.1/project/github/exarkun/tahoe-lafs/tree/2929.circleci diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh new file mode 100755 index 000000000..38ffba2a6 --- /dev/null +++ b/.circleci/run-tests.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Run the test suite as a non-root user. This is the expected usage some +# small areas of the test suite assume non-root privileges (such as unreadable +# files being unreadable). +# +# Also run with /tmp as a workdir because the non-root user won't be able to +# create the tox working filesystem state in the source checkout because it is +# owned by root. +sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e ${TAHOE_LAFS_TOX_ENVIRONMENT} diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh new file mode 100755 index 000000000..3da04a30a --- /dev/null +++ b/.circleci/setup-virtualenv.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Set up the virtualenv as a non-root user so we can run the test suite as a +# non-root user. See below. +sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests +sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov +# Get everything installed in it, too. +sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}