Merge remote-tracking branch 'origin/master' into 2932.encourage-zcash-workflow

.circleci/bootstrap-test-environment.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -e
+
+PROJECT=$1
+shift
+
+EXTRA_PACKAGES=$1
+shift
+
+# Avoid the /nonexistent home directory in nobody's /etc/passwd entry.
+usermod --home /tmp/nobody nobody
+
+# Grant read access to nobody, the user which will eventually try to test this
+# checkout.
+mv "${PROJECT}" /tmp/project
+
+# Python build/install toolchain wants to write to the source checkout, too.
+chown --recursive nobody:nogroup /tmp/project
+
+apt-get --quiet --yes install \
+    sudo \
+    build-essential \
+    python2.7 \
+    python2.7-dev \
+    libffi-dev \
+    libssl-dev \
+    libyaml-dev \
+    ${EXTRA_PACKAGES}

.circleci/config.yml

@@ -0,0 +1,298 @@
+# https://circleci.com/docs/2.0/
+
+version: 2
+workflows:
+  version: 2
+  ci:
+    jobs:
+      # Platforms
+      - "debian-9"
+      - "debian-8":
+          requires:
+            - "debian-9"
+
+      - "ubuntu-18.04"
+      - "ubuntu-16.04":
+          requires:
+            - "ubuntu-18.04"
+
+      - "fedora-28"
+      - "fedora-27":
+          requires:
+            - "fedora-28"
+
+      - "centos-7"
+
+      - "slackware-14.2"
+
+      # Other assorted tasks and configurations
+      - "lint"
+      - "deprecations"
+      - "c-locale"
+
+jobs:
+  lint:
+    docker:
+      - image: "circleci/python:2"
+
+    steps:
+      - "checkout"
+
+      - run:
+          name: "Install tox"
+          command: |
+            pip install --user tox
+
+      - run:
+          name: "Static-ish code checks"
+          command: |
+            ~/.local/bin/tox -e codechecks
+
+  debian-8: &DEBIAN
+    docker:
+      - image: "debian:8"
+
+    environment: &UTF_8_ENVIRONMENT
+      # Tell Hypothesis which configuration we want it to use.
+      TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci"
+      # Tell the C runtime things about character encoding (mainly to do with
+      # filenames and argv).
+      LANG: "en_US.UTF-8"
+      # The package name for this software varies somewhat across distros.
+      EXTRA_PACKAGES: "virtualenv"
+      # Select a tox environment to run for this job.
+      TAHOE_LAFS_TOX_ENVIRONMENT: "coverage"
+      # Additional arguments to pass to tox.
+      TAHOE_LAFS_TOX_ARGS: ""
+
+    steps:
+      - run: &INSTALL_GIT
+          node: "Install Git"
+          command: |
+            apt-get --quiet update
+            apt-get --quiet --yes install git
+
+      - "checkout"
+
+      - run: &BOOTSTRAP_TEST_ENVIRONMENT
+          name: "Bootstrap test environment"
+          command: |
+            ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}"
+
+      - run: &SETUP_VIRTUALENV
+          name: "Setup virtualenv"
+          # pip cannot install packages if the working directory is not
+          # readable.
+          working_directory: "/tmp"
+          command: |
+            env
+            /tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
+
+      - run: &RUN_TESTS
+          name: "Run test suite"
+          # Something about when it re-uses an existing environment blows up
+          # if the working directory is not readable.
+          working_directory: "/tmp"
+          command: |
+            env
+            /tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}"
+
+      - store_artifacts: &STORE_TEST_LOG
+          # Despite passing --workdir /tmp to tox above, it still runs trial
+          # in the project source checkout.
+          path: "/tmp/project/_trial_temp/test.log"
+
+      - store_artifacts: &STORE_OTHER_ARTIFACTS
+          # Store any other artifacts, too.  This is handy to allow other jobs
+          # sharing most of the definition of this one to be able to
+          # contribute artifacts easily.
+          path: "/tmp/artifacts"
+
+      - run: &SUBMIT_COVERAGE
+          name: "Submit coverage results"
+          working_directory: "/tmp/project"
+          command: |
+            /tmp/tests/bin/codecov
+
+
+  debian-9:
+    <<: *DEBIAN
+    docker:
+      - image: "debian:9"
+
+
+  c-locale:
+    <<: *DEBIAN
+    docker:
+      - image: "debian:9"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      LANG: "C"
+
+
+  deprecations:
+    <<: *DEBIAN
+    docker:
+      - image: "debian:9"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      # Select the deprecations tox environments.
+      TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations"
+      # Put the logs somewhere we can report them.
+      TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log"
+
+
+  ubuntu-16.04:
+    <<: *DEBIAN
+    docker:
+      - image: "ubuntu:16.04"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      # Necessary for en_US LANG setting.
+      EXTRA_PACKAGES: "virtualenv language-pack-en"
+
+
+  ubuntu-18.04:
+    <<: *DEBIAN
+    docker:
+      - image: "ubuntu:18.04"
+
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      # Necessary for automatic address detection/assignment.
+      EXTRA_PACKAGES: "virtualenv iproute2"
+
+
+  centos-7: &RHEL_DERIV
+    docker:
+      - image: "centos:7"
+
+    environment: *UTF_8_ENVIRONMENT
+
+    steps:
+      - run:
+          node: "Install Git"
+          command: |
+            yum install --assumeyes git
+
+      - "checkout"
+
+      - run:
+          name: "Bootstrap test environment"
+          working_directory: "/tmp"
+          command: |
+            # Avoid the /nonexistent home directory in nobody's /etc/passwd
+            # entry.
+            usermod --home /tmp/nobody nobody
+
+            # Grant read access to nobody, the user which will eventually try
+            # to test this checkout.
+            mv /root/project /tmp/project
+
+            # Python build/install toolchain wants to write to the source
+            # checkout, too.
+            chown --recursive nobody:nobody /tmp/project
+
+            yum install --assumeyes \
+                sudo \
+                make automake gcc gcc-c++ \
+                python \
+                python-devel \
+                libffi-devel \
+                openssl-devel \
+                libyaml-devel \
+                python-virtualenv
+
+            # XXX net-tools is actually a Tahoe-LAFS runtime dependency!
+            yum install --assumeyes \
+                net-tools
+
+      - run: *SETUP_VIRTUALENV
+      - run: *RUN_TESTS
+
+      - store_artifacts: *STORE_TEST_LOG
+      - store_artifacts: *STORE_OTHER_ARTIFACTS
+      - run: *SUBMIT_COVERAGE
+
+
+  fedora-27:
+    <<: *RHEL_DERIV
+    docker:
+      - image: "fedora:27"
+
+
+  fedora-28:
+    <<: *RHEL_DERIV
+    docker:
+      - image: "fedora"
+
+
+  slackware-14.2:
+    docker:
+      - image: "vbatts/slackware:14.2"
+
+    environment: *UTF_8_ENVIRONMENT
+
+    steps:
+      - run:
+          node: "Install Git"
+          command: |
+            slackpkg update
+            slackpkg install openssh-7.4p1 git-2.14.4 </dev/null
+
+      - "checkout"
+
+      - run:
+          name: "Bootstrap test environment"
+          working_directory: "/tmp"
+          command: |
+            # Avoid the /nonexistent home directory in nobody's /etc/passwd
+            # entry.
+            usermod --home /tmp/nobody nobody
+
+            # Grant read access to nobody, the user which will eventually try
+            # to test this checkout.
+            mv /root/project /tmp/project
+
+            # Python build/install toolchain wants to write to the source
+            # checkout, too.
+            chown --recursive nobody:nobody /tmp/project
+
+            slackpkg install \
+                sudo-1.8.20p2 \
+                make-4.1 \
+                automake-1.15 \
+                kernel-headers \
+                glibc-2.23 \
+                binutils-2.26 \
+                gcc-5.5.0 \
+                gcc-g++-5.5.0 \
+                python-2.7.15 \
+                libffi-3.2.1 \
+                libyaml-0.1.6 \
+                sqlite-3.13.0 \
+                icu4c-56.1 \
+                libmpc-1.0.3 </dev/null
+
+            slackpkg upgrade \
+                openssl-1.0 </dev/null
+
+            # neither virtualenv nor pip is packaged.
+            # do it the hard way.
+            # and it is extra hard since it is slackware.
+            slackpkg install \
+                cyrus-sasl-2.1.26 \
+                curl-7.60 </dev/null
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+            python get-pip.py
+            pip install virtualenv
+
+      - run: *SETUP_VIRTUALENV
+      - run: *RUN_TESTS
+
+      - store_artifacts: *STORE_TEST_LOG
+      - store_artifacts: *STORE_OTHER_ARTIFACTS
+      - run: *SUBMIT_COVERAGE

.circleci/run-build-locally.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CIRCLE_TOKEN=efb53124be82dd4b3153bc0e3f60de71da629d59
+
+curl --user ${CIRCLE_TOKEN}: \
+    --request POST \
+    --form revision=$(git rev-parse HEAD) \
+    --form config=@config.yml \
+    --form notify=false \
+    https://circleci.com/api/v1.1/project/github/exarkun/tahoe-lafs/tree/2929.circleci

.circleci/run-tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash -e
+
+TAHOE_LAFS_TOX_ENVIRONMENT=$1
+shift
+
+TAHOE_LAFS_TOX_ARGS=$1
+shift || :
+
+# Run the test suite as a non-root user.  This is the expected usage some
+# small areas of the test suite assume non-root privileges (such as unreadable
+# files being unreadable).
+#
+# Also run with /tmp as a workdir because the non-root user won't be able to
+# create the tox working filesystem state in the source checkout because it is
+# owned by root.
+sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}

.circleci/setup-virtualenv.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+TAHOE_LAFS_TOX_ENVIRONMENT=$1
+shift
+
+TAHOE_LAFS_TOX_ARGS=$1
+shift || :
+
+# Set up the virtualenv as a non-root user so we can run the test suite as a
+# non-root user.  See below.
+sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests
+
+# Slackware has non-working SSL support in setuptools until certifi is
+# installed.  SSL support in setuptools is needed in case packages use
+# `setup_requires` which gets satisfied by setuptools instead of by pip.
+# txi2p (vcversioner) is one such package.  Twisted (incremental) is another.
+sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov
+
+# Get everything else installed in it, too.
+sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS}

README.rst

@@ -10,7 +10,7 @@ function correctly, preserving your privacy and security.
 For full documentation, please see
 http://tahoe-lafs.readthedocs.io/en/latest/ .
 
-|readthedocs|  |travis|  |codecov|
+|readthedocs|  |travis|  |circleci|  |codecov|
 
 INSTALLING
 ==========
@@ -99,6 +99,9 @@ slides.
     :alt: build status
     :target: https://travis-ci.org/tahoe-lafs/tahoe-lafs
 
+.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
+    :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
+
 .. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
     :alt: test coverage percentage
     :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master

src/allmydata/test/test_util.py

@@ -643,9 +643,31 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase):
         self.failIfEqual(new_mode, 0766)
 
     def test_create_long_path(self):
+        """
+        Even for paths with total length greater than 260 bytes,
+        ``fileutil.abspath_expanduser_unicode`` produces a path on which other
+        path-related APIs can operate.
+
+        https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
+        documents certain Windows-specific path length limitations this test
+        is specifically intended to demonstrate can be overcome.
+        """
         workdir = u"test_create_long_path"
         fileutil.make_dirs(workdir)
-        long_path = fileutil.abspath_expanduser_unicode(os.path.join(workdir, u'x'*255))
+        base_path = fileutil.abspath_expanduser_unicode(workdir)
+        base_length = len(base_path)
+
+        # Construct a path /just/ long enough to exercise the important case.
+        # It would be nice if we could just use a seemingly globally valid
+        # long file name (the `x...` portion) here - for example, a name 255
+        # bytes long- and a previous version of this test did just that.
+        # However, aufs imposes a 242 byte length limit on file names.  Most
+        # other POSIX filesystems do allow names up to 255 bytes.  It's not
+        # clear there's anything we can *do* about lower limits, though, and
+        # POSIX.1-2017 (and earlier) only requires that the maximum be at
+        # least 14 (!!!)  bytes.
+        long_path = os.path.join(base_path, u'x' * (261 - base_length))
+
         def _cleanup():
             fileutil.remove(long_path)
         self.addCleanup(_cleanup)

tox.ini

@@ -14,18 +14,18 @@ skipsdist = True
 [testenv]
 basepython=python2.7
 passenv = TAHOE_LAFS_* USERPROFILE HOMEDRIVE HOMEPATH
-# Pre-install "incremental" to avoid bug #2913. Basically if Twisted's
-# setup_requires=["incremental"] causes this to be installed (with
-# setuptools), then it'll fail on travis's OS-X 10.12 machines when PyPI
-# disables access with TLS-1.1 or older, so we have to install it ahead of
-# time (with pip).
-deps = incremental
+# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
+# a package to be installed (with setuptools) then it'll fail on certain
+# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
+# requirements (TLS >= 1.2) are incompatible with the old TLS clients
+# available to those systems.  Installing it ahead of time (with pip) avoids
+# this problem.
+deps = certifi
 # We add usedevelop=True for speed, and extras=test to get things like "mock"
 # that are required for our unit tests.
 usedevelop = True
 extras = test
 commands =
-         pyflakes src static misc setup.py
          tahoe --version
          trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
 
@@ -37,7 +37,6 @@ commands =
 [testenv:coverage]
 # coverage (with --branch) takes about 65% longer to run
 commands =
-         pyflakes src static misc setup.py
          tahoe --version
          coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
          coverage xml
@@ -49,15 +48,12 @@ commands =
          python misc/coding_tools/check-debugging.py
          python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
          python misc/coding_tools/check-miscaptures.py
-         # note: check-interfaces.py imports everything, so it must be run
-         # from a populated virtualenv
-         python misc/coding_tools/check-interfaces.py
 
 [testenv:deprecations]
 setenv =
          PYTHONWARNINGS=default::DeprecationWarning
 commands =
-         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
 
 [testenv:upcoming-deprecations]
 setenv =
@@ -67,7 +63,7 @@ deps =
      git+https://github.com/warner/foolscap
 commands =
          flogtool --version
-         python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
+         python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
 
 [testenv:checkmemory]
 commands =