Merge pull request #537 from tahoe-lafs/2959.travis-trusty-tor

Move integration tests from TravisCI to CircleCI

Fixes: ticket:2959

.circleci/config.yml

@@ -30,6 +30,13 @@ workflows:
       - "deprecations"
       - "c-locale"
 
+      - "integration":
+          requires:
+            # If the unit test suite doesn't pass, don't bother running the
+            # integration tests.
+            - "debian-9"
+
+
 jobs:
   lint:
     docker:
@@ -48,9 +55,9 @@ jobs:
           command: |
             ~/.local/bin/tox -e codechecks
 
-  debian-8: &DEBIAN
+  debian-9: &DEBIAN
     docker:
-      - image: "debian:8"
+      - image: "debian:9"
 
     environment: &UTF_8_ENVIRONMENT
       # Tell Hypothesis which configuration we want it to use.
@@ -64,6 +71,8 @@ jobs:
       TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
+      # The path in which test artifacts will be placed.
+      ARTIFACTS_OUTPUT_PATH: "/tmp/artifacts"
       # Convince all of our pip invocations to look at the cached wheelhouse
       # we maintain.
       WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
@@ -145,7 +154,7 @@ jobs:
           working_directory: "/tmp"
           command: |
             /tmp/project/.circleci/run-tests.sh \
-                /tmp/artifacts \
+                "${ARTIFACTS_OUTPUT_PATH}" \
                 "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
                 "${TAHOE_LAFS_TOX_ARGS}"
           # trial output gets directed straight to a log.  avoid the circleci
@@ -173,16 +182,14 @@ jobs:
             /tmp/tests/bin/codecov
 
 
-  debian-9:
+  debian-8:
     <<: *DEBIAN
     docker:
-      - image: "debian:9"
+      - image: "debian:8"
 
 
   c-locale:
     <<: *DEBIAN
-    docker:
-      - image: "debian:9"
 
     environment:
       <<: *UTF_8_ENVIRONMENT
@@ -191,8 +198,6 @@ jobs:
 
   deprecations:
     <<: *DEBIAN
-    docker:
-      - image: "debian:9"
 
     environment:
       <<: *UTF_8_ENVIRONMENT
@@ -202,6 +207,35 @@ jobs:
       TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
 
 
+  integration:
+    <<: *DEBIAN
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      # Select the integration tests tox environments.
+      TAHOE_LAFS_TOX_ENVIRONMENT: "integration"
+      # Disable artifact collection because py.test can't produce any.
+      ARTIFACTS_OUTPUT_PATH: ""
+
+
+    steps:
+      # DRY, YAML-style.  See the debian-9 steps.
+      - run: *INSTALL_GIT
+      - "checkout"
+      - run: *BOOTSTRAP_TEST_ENVIRONMENT
+      - restore_cache: *RESTORE_HTTP_CACHE
+      - restore_cache: *RESTORE_WHEELHOUSE
+      - run: *SETUP_VIRTUALENV
+      - run:
+          name: "Install Tor"
+          command: |
+            # Don't forget that we moved the source tree to /tmp.
+            /tmp/project/integration/install-tor.sh
+      - save_cache: *SAVE_HTTP_CACHE
+      - save_cache: *SAVE_WHEELHOUSE
+      - run: *RUN_TESTS
+
+
   ubuntu-16.04:
     <<: *DEBIAN
     docker:

.circleci/run-tests.sh

@@ -9,14 +9,20 @@ shift
 TAHOE_LAFS_TOX_ARGS=$1
 shift || :
 
-# Make sure we can actually write things to this directory.
-sudo --user nobody mkdir -p "${ARTIFACTS}"
+if [ -n "${ARTIFACTS}" ]; then
+    # If given an artifacts path, prepare to have some artifacts created
+    # there.  The integration tests don't produce any artifacts; that is the
+    # case where we expect not to end up here.
 
-SUBUNIT2="${ARTIFACTS}"/results.subunit2
+    # Make sure we can actually write things to this directory.
+    sudo --user nobody mkdir -p "${ARTIFACTS}"
 
-# Use an intermediate directory here because CircleCI extracts some label
-# information from its name.
-JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
+    SUBUNIT2="${ARTIFACTS}"/results.subunit2
+
+    # Use an intermediate directory here because CircleCI extracts some label
+    # information from its name.
+    JUNITXML="${ARTIFACTS}"/junit/unittests/results.xml
+fi
 
 # Run the test suite as a non-root user.  This is the expected usage some
 # small areas of the test suite assume non-root privileges (such as unreadable
@@ -40,6 +46,8 @@ sudo \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
     ${TAHOE_LAFS_TOX_ARGS}
 
-# Create a junitxml results area.
-mkdir -p "$(dirname "${JUNITXML}")"
-/tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
+if [ -n "${ARTIFACTS}" ]; then
+    # Create a junitxml results area.
+    mkdir -p "$(dirname "${JUNITXML}")"
+    /tmp/tests/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}"
+fi

.travis.yml

@@ -22,7 +22,6 @@ install:
   - python misc/build_helpers/show-tool-versions.py
 
 script:
-  - if [ "${T}" = "integration" ]; then /bin/bash integration/install-tor.sh; fi
   - tox -e ${T}
 
 after_success:
@@ -42,8 +41,6 @@ matrix:
   include:
   - os: linux
     env: T=coverage LANG=en_US.UTF-8
-  - os: linux
-    env: T=integration LANG=en_US.UTF-8
   - os: linux
     env: T=codechecks LANG=en_US.UTF-8
   - os: linux

integration/conftest.py

@@ -10,7 +10,10 @@ from tempfile import mkdtemp, mktemp
 from twisted.python.procutils import which
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.internet.task import deferLater
-from twisted.internet.error import ProcessExitedAlready
+from twisted.internet.error import (
+    ProcessExitedAlready,
+    ProcessTerminated,
+)
 
 import pytest
 
@@ -414,6 +417,8 @@ def chutney(reactor, temp_dir):
     # XXX yuck! should add a setup.py to chutney so we can at least
     # "pip install <path to tarball>" and/or depend on chutney in "pip
     # install -e .[dev]" (i.e. in the 'dev' extra)
+    #
+    # https://trac.torproject.org/projects/tor/ticket/20343
     proto = _DumpOutputProtocol(None)
     reactor.spawnProcess(
         proto,
@@ -476,7 +481,11 @@ def tor_network(reactor, temp_dir, chutney, request):
         path=join(chutney_dir),
         env={"PYTHONPATH": join(chutney_dir, "lib")},
     )
-    pytest.blockon(proto.done)
+    try:
+        pytest.blockon(proto.done)
+    except ProcessTerminated:
+        print("Chutney.TorNet status failed (continuing):")
+        print(proto.output.getvalue())
 
     def cleanup():
         print("Tearing down Chutney Tor network")

integration/install-tor.sh

@@ -3,8 +3,8 @@
 # Script to install Tor
 
 set -ex
-echo "deb http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list
-echo "deb-src http://deb.torproject.org/torproject.org trusty main" | sudo tee -a /etc/apt/sources.list
+echo "deb http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
+echo "deb-src http://deb.torproject.org/torproject.org stretch main" | sudo tee -a /etc/apt/sources.list
 
 # Install Tor repo signing key
 sudo apt-key add - <<EOF
@@ -707,5 +707,5 @@ I4Tesw==
 -----END PGP PUBLIC KEY BLOCK-----
 EOF
 
-sudo apt-get update
-sudo apt-get install tor deb.torproject.org-keyring
+sudo apt-get --quiet update
+sudo apt-get --quiet --yes install tor deb.torproject.org-keyring

newsfragments/2959.other

@@ -0,0 +1 @@
+Some CI jobs for integration tests have been moved from TravisCI to CircleCI.

src/allmydata/frontends/magic_folder.py

@@ -7,6 +7,7 @@ from datetime import datetime
 import time
 import ConfigParser
 
+from twisted.python.monkey import MonkeyPatcher
 from twisted.internet import defer, reactor, task
 from twisted.internet.error import AlreadyCancelled
 from twisted.python.failure import Failure
@@ -48,7 +49,7 @@ class ConfigurationError(Exception):
     """
 
 
-def get_inotify_module():
+def _get_inotify_module():
     try:
         if sys.platform == "win32":
             from allmydata.windows import inotify
@@ -66,6 +67,14 @@ def get_inotify_module():
         raise
 
 
+def get_inotify_module():
+    # Until Twisted #9579 is fixed, the Docker check just screws things up.
+    # Disable it.
+    monkey = MonkeyPatcher()
+    monkey.addPatch(runtime.platform, "isDocker", lambda: False)
+    return monkey.runWithPatches(_get_inotify_module)
+
+
 def is_new_file(pathinfo, db_entry):
     if db_entry is None:
         return True