Merge pull request #509 from LeastAuthority/circleci-store-test-results

Provide detailed test results to CircleCI.

This enables improved tracking and reporting in the CircleCI UI.
This commit is contained in:
Jean-Paul Calderone 2018-07-09 14:31:21 -04:00 committed by GitHub
commit f12c3cd44c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 74 additions and 13 deletions

View File

@ -85,8 +85,9 @@ jobs:
# readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
/tmp/project/.circleci/setup-virtualenv.sh \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
- run: &RUN_TESTS
name: "Run test suite"
@ -94,8 +95,16 @@ jobs:
# if the working directory is not readable.
working_directory: "/tmp"
command: |
env
/tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
/tmp/project/.circleci/run-tests.sh \
/tmp/artifacts \
"${TAHOE_LAFS_TOX_ENVIRONMENT}" \
"${TAHOE_LAFS_TOX_ARGS}"
# trial output gets directed straight to a log. avoid the circleci
# timeout while the test suite runs.
no_output_timeout: "20m"
- store_test_results: &STORE_TEST_RESULTS
path: "/tmp/artifacts/junit"
- store_artifacts: &STORE_TEST_LOG
# Despite passing --workdir /tmp to tox above, it still runs trial
@ -213,6 +222,7 @@ jobs:
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE
@ -262,6 +272,7 @@ jobs:
chown --recursive nobody:nobody /tmp/project
slackpkg install \
ca-certificates \
sudo-1.8.20p2 \
make-4.1 \
automake-1.15 \
@ -293,6 +304,7 @@ jobs:
- run: *SETUP_VIRTUALENV
- run: *RUN_TESTS
- store_test_results: *STORE_TEST_RESULTS
- store_artifacts: *STORE_TEST_LOG
- store_artifacts: *STORE_OTHER_ARTIFACTS
- run: *SUBMIT_COVERAGE

View File

@ -1,11 +1,23 @@
#!/bin/bash -e
ARTIFACTS=$1
shift
TAHOE_LAFS_TOX_ENVIRONMENT=$1
shift
TAHOE_LAFS_TOX_ARGS=$1
shift || :
# Make sure we can actually write things to this directory.
sudo --user nobody mkdir -p "${ARTIFACTS}"
SUBUNIT2="${ARTIFACTS}"/results.subunit2
# Use an intermediate directory here because CircleCI extracts some label
# information from its name.
JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
# Run the test suite as a non-root user. This is the expected usage some
# small areas of the test suite assume non-root privileges (such as unreadable
# files being unreadable).
@ -13,4 +25,20 @@ shift || :
# Also run with /tmp as a workdir because the non-root user won't be able to
# create the tox working filesystem state in the source checkout because it is
# owned by root.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
#
# Send the output directly to a file because transporting the binary subunit2
# via tox and then scraping it out is hideous and failure prone.
sudo \
SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}" \
TAHOE_LAFS_TRIAL_ARGS="--reporter=subunitv2-file" \
--set-home \
--user nobody \
/tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS}
# Create a junitxml results area.
mkdir -p "$(dirname "${JUNITXML}")"
/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"

View File

@ -10,11 +10,28 @@ shift || :
# non-root user. See below.
sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
# Slackware has non-working SSL support in setuptools until certifi is
# installed. SSL support in setuptools is needed in case packages use
# `setup_requires` which gets satisfied by setuptools instead of by pip.
# txi2p (vcversioner) is one such package. Twisted (incremental) is another.
sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem.
sudo --set-home -u nobody /tmp/tests/bin/pip install certifi
# Python packages we need to support the test infrastructure. *Not* packages
# Tahoe-LAFS itself (implementation or test suite) need.
TEST_DEPS="tox codecov"
# Python packages we need to generate test reports for CI infrastructure.
# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
REPORTING_DEPS="python-subunit junitxml subunitreporter"
sudo --set-home -u nobody /tmp/tests/bin/pip install ${TEST_DEPS} ${REPORTING_DEPS}
# Get everything else installed in it, too.
sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}
sudo --set-home -u nobody /tmp/tests/bin/tox \
-c /tmp/project/tox.ini \
--workdir /tmp/tahoe-lafs.tox \
--notest \
-e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
${TAHOE_LAFS_TOX_ARGS}

View File

@ -13,14 +13,16 @@ skipsdist = True
[testenv]
basepython=python2.7
passenv = TAHOE_LAFS_* USERPROFILE HOMEDRIVE HOMEPATH
passenv = TAHOE_LAFS_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
# a package to be installed (with setuptools) then it'll fail on certain
# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
# requirements (TLS >= 1.2) are incompatible with the old TLS clients
# available to those systems. Installing it ahead of time (with pip) avoids
# this problem.
deps = certifi
deps =
certifi
subunitreporter
# We add usedevelop=True for speed, and extras=test to get things like "mock"
# that are required for our unit tests.
usedevelop = True
@ -59,6 +61,8 @@ commands =
setenv =
PYTHONWARNINGS=default::DeprecationWarning
deps =
# Take the base deps as well!
{[testenv]deps}
git+https://github.com/twisted/twisted
git+https://github.com/warner/foolscap
commands =