diff --git a/.circleci/config.yml b/.circleci/config.yml index f041dabdf..9c8c32ccb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,13 @@ workflows: - "deprecations" - "c-locale" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-9" + + jobs: lint: docker: @@ -48,9 +55,9 @@ jobs: command: | ~/.local/bin/tox -e codechecks - debian-8: &DEBIAN + debian-9: &DEBIAN docker: - - image: "debian:8" + - image: "debian:9" environment: &UTF_8_ENVIRONMENT # Tell Hypothesis which configuration we want it to use. @@ -64,6 +71,8 @@ jobs: TAHOE_LAFS_TOX_ENVIRONMENT: "coverage" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" + # The path in which test artifacts will be placed. + ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts" # Convince all of our pip invocations to look at the cached wheelhouse # we maintain. WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse" @@ -145,7 +154,7 @@ jobs: working_directory: "/tmp" command: | /tmp/project/.circleci/run-tests.sh \ - /tmp/artifacts \ + "${ARTIFACTS_OUTPUT_PATH}" \ "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ "${TAHOE_LAFS_TOX_ARGS}" # trial output gets directed straight to a log. avoid the circleci @@ -173,16 +182,14 @@ jobs: /tmp/tests/bin/codecov - debian-9: + debian-8: <<: *DEBIAN docker: - - image: "debian:9" + - image: "debian:8" c-locale: <<: *DEBIAN - docker: - - image: "debian:9" environment: <<: *UTF_8_ENVIRONMENT @@ -191,8 +198,6 @@ jobs: deprecations: <<: *DEBIAN - docker: - - image: "debian:9" environment: <<: *UTF_8_ENVIRONMENT @@ -202,6 +207,35 @@ jobs: TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log" + integration: + <<: *DEBIAN + + environment: + <<: *UTF_8_ENVIRONMENT + # Select the integration tests tox environments. + TAHOE_LAFS_TOX_ENVIRONMENT: "integration" + # Disable artifact collection because py.test can't produce any. + ARTIFACTS_OUTPUT_PATH: "" + + + steps: + # DRY, YAML-style. See the debian-9 steps. + - run: *INSTALL_GIT + - "checkout" + - run: *BOOTSTRAP_TEST_ENVIRONMENT + - restore_cache: *RESTORE_HTTP_CACHE + - restore_cache: *RESTORE_WHEELHOUSE + - run: *SETUP_VIRTUALENV + - run: + name: "Install Tor" + command: | + # Don't forget that we moved the source tree to /tmp. + /tmp/project/integration/install-tor.sh + - save_cache: *SAVE_HTTP_CACHE + - save_cache: *SAVE_WHEELHOUSE + - run: *RUN_TESTS + + ubuntu-16.04: <<: *DEBIAN docker: diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 6b61899e5..0b040782d 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -9,14 +9,20 @@ shift TAHOE_LAFS_TOX_ARGS=$1 shift || : -# Make sure we can actually write things to this directory. -sudo --user nobody mkdir -p "${ARTIFACTS}" +if [ -n "${ARTIFACTS}" ]; then + # If given an artifacts path, prepare to have some artifacts created + # there. The integration tests don't produce any artifacts; that is the + # case where we expect not to end up here. -SUBUNIT2="${ARTIFACTS}"/results.subunit2 + # Make sure we can actually write things to this directory. + sudo --user nobody mkdir -p "${ARTIFACTS}" -# Use an intermediate directory here because CircleCI extracts some label -# information from its name. -JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml + SUBUNIT2="${ARTIFACTS}"/results.subunit2 + + # Use an intermediate directory here because CircleCI extracts some label + # information from its name. + JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml +fi # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable @@ -40,6 +46,8 @@ sudo \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ ${TAHOE_LAFS_TOX_ARGS} -# Create a junitxml results area. -mkdir -p "$(dirname "${JUNITXML}")" -/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" +if [ -n "${ARTIFACTS}" ]; then + # Create a junitxml results area. + mkdir -p "$(dirname "${JUNITXML}")" + /tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" +fi diff --git a/.travis.yml b/.travis.yml index 10154d53f..2cac94c00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ install: - python misc/build_helpers/show-tool-versions.py script: - - if [ "${T}" = "integration" ]; then /bin/bash integration/install-tor.sh; fi - tox -e ${T} after_success: @@ -42,8 +41,6 @@ matrix: include: - os: linux env: T=coverage LANG=en_US.UTF-8 - - os: linux - env: T=integration LANG=en_US.UTF-8 - os: linux env: T=codechecks LANG=en_US.UTF-8 - os: linux diff --git a/integration/conftest.py b/integration/conftest.py index ca9d55e8c..34b1b4418 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -10,7 +10,10 @@ from tempfile import mkdtemp, mktemp from twisted.python.procutils import which from twisted.internet.defer import Deferred, DeferredList from twisted.internet.task import deferLater -from twisted.internet.error import ProcessExitedAlready +from twisted.internet.error import ( + ProcessExitedAlready, + ProcessTerminated, +) import pytest @@ -414,6 +417,8 @@ def chutney(reactor, temp_dir): # XXX yuck! should add a setup.py to chutney so we can at least # "pip install " and/or depend on chutney in "pip # install -e .[dev]" (i.e. in the 'dev' extra) + # + # https://trac.torproject.org/projects/tor/ticket/20343 proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -476,7 +481,11 @@ def tor_network(reactor, temp_dir, chutney, request): path=join(chutney_dir), env={"PYTHONPATH": join(chutney_dir, "lib")}, ) - pytest.blockon(proto.done) + try: + pytest.blockon(proto.done) + except ProcessTerminated: + print("Chutney.TorNet status failed (continuing):") + print(proto.output.getvalue()) def cleanup(): print("Tearing down Chutney Tor network") diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 3cfe27899..47f08391e 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -3,8 +3,8 @@ # Script to install Tor set -ex -echo "deb http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list -echo "deb-src http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list +echo "deb http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list +echo "deb-src http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list # Install Tor repo signing key sudo apt-key add - <