From 9c359681f257903903e6085b08f21b754ca019e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Jan 2019 18:15:25 -0500 Subject: [PATCH 01/14] Flip Debian 8 and 9 definitions More things want Debian 9. This reduces duplication. --- .circleci/config.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f041dabdf..5404d7ee7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,9 +48,9 @@ jobs: command: | ~/.local/bin/tox -e codechecks - debian-8: &DEBIAN + debian-9: &DEBIAN docker: - - image: "debian:8" + - image: "debian:9" environment: &UTF_8_ENVIRONMENT # Tell Hypothesis which configuration we want it to use. @@ -173,16 +173,14 @@ jobs: /tmp/tests/bin/codecov - debian-9: + debian-8: <<: *DEBIAN docker: - - image: "debian:9" + - image: "debian:8" c-locale: <<: *DEBIAN - docker: - - image: "debian:9" environment: <<: *UTF_8_ENVIRONMENT @@ -191,8 +189,6 @@ jobs: deprecations: <<: *DEBIAN - docker: - - image: "debian:9" environment: <<: *UTF_8_ENVIRONMENT From f534d3c34fd9e16f53413dc19d0a35538f797671 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 09:02:27 -0500 Subject: [PATCH 02/14] Attempt to configure CircleCI for integration tests --- .circleci/config.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5404d7ee7..8ffe14a25 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,7 @@ workflows: - "lint" - "deprecations" - "c-locale" + - "integration" jobs: lint: @@ -198,6 +199,34 @@ jobs: TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log" + integration: + <<: *DEBIAN + + environment: + <<: *UTF_8_ENVIRONMENT + # Select the integration tests tox environments. + TAHOE_LAFS_TOX_ENVIRONMENT: "integration" + + steps: + # DRY, YAML-style. See the debian-9 steps. + - run: *INSTALL_GIT + - "checkout" + - run: *BOOTSTRAP_TEST_ENVIRONMENT + - restore_cache: *RESTORE_HTTP_CACHE + - restore_cache: *RESTORE_WHEELHOUSE + - run: *SETUP_VIRTUALENV + - run: + name: "Install Tor" + command: | + integration/install-tor.sh + - save_cache: *SAVE_HTTP_CACHE + - save_cache: *SAVE_WHEELHOUSE + - run: *RUN_TESTS + - store_test_results: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE + + ubuntu-16.04: <<: *DEBIAN docker: From 3c5cb8d3cf49825c7b87ab422f79e2cfebaccfaa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 09:03:06 -0500 Subject: [PATCH 03/14] deconfigure travisci for integration testing --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10154d53f..2cac94c00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ install: - python misc/build_helpers/show-tool-versions.py script: - - if [ "${T}" = "integration" ]; then /bin/bash integration/install-tor.sh; fi - tox -e ${T} after_success: @@ -42,8 +41,6 @@ matrix: include: - os: linux env: T=coverage LANG=en_US.UTF-8 - - os: linux - env: T=integration LANG=en_US.UTF-8 - os: linux env: T=codechecks LANG=en_US.UTF-8 - os: linux From 25719ca1cbdbc723937d8b41c017c4a4f5d30eb3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 10:04:07 -0500 Subject: [PATCH 04/14] Invoke install-tor.sh with the correct path. --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ffe14a25..07d29280b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,7 +218,8 @@ jobs: - run: name: "Install Tor" command: | - integration/install-tor.sh + # Don't forget that we moved the source tree to /tmp. + /tmp/project/integration/install-tor.sh - save_cache: *SAVE_HTTP_CACHE - save_cache: *SAVE_WHEELHOUSE - run: *RUN_TESTS From a7bf6b2264955769cd02bbedd12c51b52b4dbac7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 10:16:50 -0500 Subject: [PATCH 05/14] Switch to Tor repo for distro we're now using --- integration/install-tor.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 3cfe27899..0a4e7d125 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -3,8 +3,8 @@ # Script to install Tor set -ex -echo "deb http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list -echo "deb-src http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list +echo "deb http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list +echo "deb-src http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list # Install Tor repo signing key sudo apt-key add - < Date: Thu, 24 Jan 2019 10:20:47 -0500 Subject: [PATCH 06/14] Block integration tests on debian-9 results --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07d29280b..ee86d1916 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,13 @@ workflows: - "lint" - "deprecations" - "c-locale" - - "integration" + + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-9" + jobs: lint: From 6e5ba22f595758f11848afd1222e4445d1f9f801 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 10:31:12 -0500 Subject: [PATCH 07/14] do as I ask --- integration/install-tor.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/install-tor.sh b/integration/install-tor.sh index 0a4e7d125..47f08391e 100755 --- a/integration/install-tor.sh +++ b/integration/install-tor.sh @@ -707,5 +707,5 @@ I4Tesw== -----END PGP PUBLIC KEY BLOCK----- EOF -sudo apt-get update -sudo apt-get install tor deb.torproject.org-keyring +sudo apt-get --quiet update +sudo apt-get --quiet --yes install tor deb.torproject.org-keyring From 7e9e447d5b2d04654b8bfed7d6676e6da9ec20c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 14:00:50 -0500 Subject: [PATCH 08/14] Disable Twisted's Docker check wrt inotify support It breaks our container-based CI, which is perfectly happy to give us inotify functionality, as is Docker in general as long as you don't use overlayfs. --- src/allmydata/frontends/magic_folder.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/allmydata/frontends/magic_folder.py b/src/allmydata/frontends/magic_folder.py index 1c6c60e23..c125a4976 100644 --- a/src/allmydata/frontends/magic_folder.py +++ b/src/allmydata/frontends/magic_folder.py @@ -7,6 +7,7 @@ from datetime import datetime import time import ConfigParser +from twisted.python.monkey import MonkeyPatcher from twisted.internet import defer, reactor, task from twisted.internet.error import AlreadyCancelled from twisted.python.failure import Failure @@ -48,7 +49,7 @@ class ConfigurationError(Exception): """ -def get_inotify_module(): +def _get_inotify_module(): try: if sys.platform == "win32": from allmydata.windows import inotify @@ -66,6 +67,14 @@ def get_inotify_module(): raise +def get_inotify_module(): + # Until Twisted #9579 is fixed, the Docker check just screws things up. + # Disable it. + monkey = MonkeyPatcher() + monkey.addPatch(runtime.platform, "isDocker", lambda: False) + return monkey.runWithPatches(_get_inotify_module) + + def is_new_file(pathinfo, db_entry): if db_entry is None: return True From bedf092ca39eb53923432bd2777df2df8f1fa127 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 14:48:09 -0500 Subject: [PATCH 09/14] Try to report the child process output --- integration/conftest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index ca9d55e8c..6bbcaa0e7 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -10,7 +10,10 @@ from tempfile import mkdtemp, mktemp from twisted.python.procutils import which from twisted.internet.defer import Deferred, DeferredList from twisted.internet.task import deferLater -from twisted.internet.error import ProcessExitedAlready +from twisted.internet.error import ( + ProcessExitedAlready, + ProcessTerminated, +) import pytest @@ -476,7 +479,12 @@ def tor_network(reactor, temp_dir, chutney, request): path=join(chutney_dir), env={"PYTHONPATH": join(chutney_dir, "lib")}, ) - pytest.blockon(proto.done) + try: + pytest.blockon(proto.done) + except ProcessTerminated: + print("Chutney.TorNet failed:") + print(proto.output.getvalue()) + raise def cleanup(): print("Tearing down Chutney Tor network") From 402b2c3bd389333137ac812db13b2a182c9d87e3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 15:40:01 -0500 Subject: [PATCH 10/14] If TorNet status fails, proceed anyway --- integration/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 6bbcaa0e7..b4e14b702 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -482,9 +482,8 @@ def tor_network(reactor, temp_dir, chutney, request): try: pytest.blockon(proto.done) except ProcessTerminated: - print("Chutney.TorNet failed:") + print("Chutney.TorNet status failed (continuing):") print(proto.output.getvalue()) - raise def cleanup(): print("Tearing down Chutney Tor network") From ab58c7a938a5017c54928af4018478cd849bc19e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 15:53:05 -0500 Subject: [PATCH 11/14] Link to the ticket for that --- integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index b4e14b702..34b1b4418 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -417,6 +417,8 @@ def chutney(reactor, temp_dir): # XXX yuck! should add a setup.py to chutney so we can at least # "pip install " and/or depend on chutney in "pip # install -e .[dev]" (i.e. in the 'dev' extra) + # + # https://trac.torproject.org/projects/tor/ticket/20343 proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, From 6ee727d67e1adf0205bf7aeb7f3be9ca545679ae Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 16:04:26 -0500 Subject: [PATCH 12/14] Avoid the artifact steps for integration tests Integration tests produce none and trying to do these things breaks. --- .circleci/config.yml | 7 ++++++- .circleci/run-tests.sh | 26 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ee86d1916..3b0c24245 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,6 +71,8 @@ jobs: TAHOE_LAFS_TOX_ENVIRONMENT: "coverage" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" + # The path in which test artifacts will be placed. + ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts" # Convince all of our pip invocations to look at the cached wheelhouse # we maintain. WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse" @@ -152,7 +154,7 @@ jobs: working_directory: "/tmp" command: | /tmp/project/.circleci/run-tests.sh \ - /tmp/artifacts \ + "${ARTIFACTS_OUTPUT_PATH}" \ "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ "${TAHOE_LAFS_TOX_ARGS}" # trial output gets directed straight to a log. avoid the circleci @@ -212,6 +214,9 @@ jobs: <<: *UTF_8_ENVIRONMENT # Select the integration tests tox environments. TAHOE_LAFS_TOX_ENVIRONMENT: "integration" + # Disable artifact collection because py.test can't produce any. + ARTIFACTS_OUTPUT_PATH: "" + steps: # DRY, YAML-style. See the debian-9 steps. diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 6b61899e5..0b040782d 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -9,14 +9,20 @@ shift TAHOE_LAFS_TOX_ARGS=$1 shift || : -# Make sure we can actually write things to this directory. -sudo --user nobody mkdir -p "${ARTIFACTS}" +if [ -n "${ARTIFACTS}" ]; then + # If given an artifacts path, prepare to have some artifacts created + # there. The integration tests don't produce any artifacts; that is the + # case where we expect not to end up here. -SUBUNIT2="${ARTIFACTS}"/results.subunit2 + # Make sure we can actually write things to this directory. + sudo --user nobody mkdir -p "${ARTIFACTS}" -# Use an intermediate directory here because CircleCI extracts some label -# information from its name. -JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml + SUBUNIT2="${ARTIFACTS}"/results.subunit2 + + # Use an intermediate directory here because CircleCI extracts some label + # information from its name. + JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml +fi # Run the test suite as a non-root user. This is the expected usage some # small areas of the test suite assume non-root privileges (such as unreadable @@ -40,6 +46,8 @@ sudo \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ ${TAHOE_LAFS_TOX_ARGS} -# Create a junitxml results area. -mkdir -p "$(dirname "${JUNITXML}")" -/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" +if [ -n "${ARTIFACTS}" ]; then + # Create a junitxml results area. + mkdir -p "$(dirname "${JUNITXML}")" + /tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" +fi From 1fb280927661a2cd0cb8e552aff9561d75c7afa5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 16:04:58 -0500 Subject: [PATCH 13/14] Additionally, disable the CircleCI artifact handling steps --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b0c24245..9c8c32ccb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -234,9 +234,6 @@ jobs: - save_cache: *SAVE_HTTP_CACHE - save_cache: *SAVE_WHEELHOUSE - run: *RUN_TESTS - - store_test_results: *STORE_TEST_LOG - - store_artifacts: *STORE_OTHER_ARTIFACTS - - run: *SUBMIT_COVERAGE ubuntu-16.04: From e23a017e2f271544614fddc701224a96686df66a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 24 Jan 2019 16:38:01 -0500 Subject: [PATCH 14/14] news fragment --- newsfragments/2959.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2959.other diff --git a/newsfragments/2959.other b/newsfragments/2959.other new file mode 100644 index 000000000..a8da6337a --- /dev/null +++ b/newsfragments/2959.other @@ -0,0 +1 @@ +Some CI jobs for integration tests have been moved from TravisCI to CircleCI.