diff --git a/.circleci/bootstrap-test-environment.sh b/.circleci/bootstrap-test-environment.sh new file mode 100755 index 000000000..193a3443b --- /dev/null +++ b/.circleci/bootstrap-test-environment.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +PROJECT=$1 +shift + +EXTRA_PACKAGES=$1 +shift + +# Avoid the /nonexistent home directory in nobody's /etc/passwd entry. +usermod --home /tmp/nobody nobody + +# Grant read access to nobody, the user which will eventually try to test this +# checkout. +mv "${PROJECT}" /tmp/project + +# Python build/install toolchain wants to write to the source checkout, too. +chown --recursive nobody:nogroup /tmp/project + +apt-get --quiet --yes install \ + sudo \ + build-essential \ + python2.7 \ + python2.7-dev \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + ${EXTRA_PACKAGES} diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..17278b047 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,261 @@ +# https://circleci.com/docs/2.0/ + +version: 2 +workflows: + version: 2 + ci: + jobs: + - "lint" + - "debian-8" + - "debian-9" + - "ubuntu-16.04" + - "ubuntu-18.04" + - "centos-7" + - "fedora-27" + - "fedora-28" + + - "magic-folder-ubuntu-14.04" + - "deprecations" + - "c-locale" + +jobs: + lint: + docker: + - image: "circleci/python:2" + + steps: + - "checkout" + + - run: + name: "Install tox" + command: | + pip install --user tox + + - run: + name: "Static-ish code checks" + command: | + ~/.local/bin/tox -e codechecks + + debian-8: &DEBIAN + docker: + - image: "debian:8" + + environment: &UTF_8_ENVIRONMENT + # Tell Hypothesis which configuration we want it to use. + TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + # Tell the C runtime things about character encoding (mainly to do with + # filenames and argv). + LANG: "en_US.UTF-8" + # The package name for this software varies somewhat across distros. + EXTRA_PACKAGES: "virtualenv" + # Select a tox environment to run for this job. + TAHOE_LAFS_TOX_ENVIRONMENT: "coverage" + # Additional arguments to pass to tox. + TAHOE_LAFS_TOX_ARGS: "" + + steps: + - run: &INSTALL_GIT + node: "Install Git" + command: | + apt-get --quiet update + apt-get --quiet --yes install git + + - "checkout" + + - run: &BOOTSTRAP_TEST_ENVIRONMENT + name: "Bootstrap test environment" + command: | + ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}" + + - run: &SETUP_VIRTUALENV + name: "Setup virtualenv" + # pip cannot install packages if the working directory is not + # readable. + working_directory: "/tmp" + command: | + env + /tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}" + + - run: &RUN_TESTS + name: "Run test suite" + # Something about when it re-uses an existing environment blows up + # if the working directory is not readable. + working_directory: "/tmp" + command: | + env + /tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}" + + - store_artifacts: &STORE_TEST_LOG + # Despite passing --workdir /tmp to tox above, it still runs trial + # in the project source checkout. + path: "/tmp/project/_trial_temp/test.log" + + - store_artifacts: &STORE_OTHER_ARTIFACTS + # Store any other artifacts, too. This is handy to allow other jobs + # sharing most of the definition of this one to be able to + # contribute artifacts easily. + path: "/tmp/artifacts" + + - run: &SUBMIT_COVERAGE + name: "Submit coverage results" + working_directory: "/tmp/project" + command: | + /tmp/tests/bin/codecov + + + debian-9: + <<: *DEBIAN + docker: + - image: "debian:9" + + + c-locale: + <<: *DEBIAN + docker: + - image: "debian:9" + + environment: + <<: *UTF_8_ENVIRONMENT + LANG: "C" + + + + deprecations: + <<: *DEBIAN + docker: + - image: "debian:9" + + environment: + <<: *UTF_8_ENVIRONMENT + # Select the deprecations tox environments. + TAHOE_LAFS_TOX_ENVIRONMENT: "deprecations,upcoming-deprecations" + # Put the logs somewhere we can report them. + TAHOE_LAFS_WARNINGS_LOG: "/tmp/artifacts/deprecation-warnings.log" + + + ubuntu-16.04: + <<: *DEBIAN + docker: + - image: "ubuntu:16.04" + + environment: + <<: *UTF_8_ENVIRONMENT + # Necessary for en_US LANG setting. + EXTRA_PACKAGES: "virtualenv language-pack-en" + + + ubuntu-18.04: + <<: *DEBIAN + docker: + - image: "ubuntu:18.04" + + environment: + <<: *UTF_8_ENVIRONMENT + # Necessary for automatic address detection/assignment. + EXTRA_PACKAGES: "virtualenv iproute2" + + + centos-7: &RHEL_DERIV + docker: + - image: "centos:7" + + environment: *UTF_8_ENVIRONMENT + + steps: + - run: + node: "Install Git" + command: | + yum install --assumeyes git + + - "checkout" + + - run: + name: "Bootstrap test environment" + working_directory: "/tmp" + command: | + # Avoid the /nonexistent home directory in nobody's /etc/passwd + # entry. + usermod --home /tmp/nobody nobody + + # Grant read access to nobody, the user which will eventually try + # to test this checkout. + mv /root/project /tmp/project + + # Python build/install toolchain wants to write to the source + # checkout, too. + chown --recursive nobody:nobody /tmp/project + + yum install --assumeyes \ + sudo \ + make automake gcc gcc-c++ \ + python \ + python-devel \ + libffi-devel \ + openssl-devel \ + libyaml-devel \ + python-virtualenv + + # XXX net-tools is actually a Tahoe-LAFS runtime dependency! + yum install --assumeyes \ + net-tools + + - run: *SETUP_VIRTUALENV + - run: *RUN_TESTS + + - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE + + + fedora-27: + <<: *RHEL_DERIV + docker: + - image: "fedora:27" + + + fedora-28: + <<: *RHEL_DERIV + docker: + - image: "fedora" + + + magic-folder-ubuntu-14.04: + machine: + enabled: true + image: "circleci/classic:201711-01" + + environment: + <<: *UTF_8_ENVIRONMENT + EXTRA_PACKAGES: "python-virtualenv" + TAHOE_LAFS_TOX_ARGS: "-- allmydata.test.test_magic_folder" + + # Unfortunately, duplicate all the steps here but run with `sudo`. + steps: + - run: + node: "Install Git" + command: | + sudo apt-get --quiet update + sudo apt-get --quiet --yes install git + + - "checkout" + + - run: + <<: *BOOTSTRAP_TEST_ENVIRONMENT + command: | + sudo ~/project/.circleci/bootstrap-test-environment.sh ~/project "${EXTRA_PACKAGES}" + + - run: + <<: *SETUP_VIRTUALENV + command: | + env + sudo /tmp/project/.circleci/setup-virtualenv.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}" + + - run: + <<: *RUN_TESTS + command: | + env + sudo /tmp/project/.circleci/run-tests.sh "${TAHOE_LAFS_TOX_ENVIRONMENT}" "${TAHOE_LAFS_TOX_ARGS}" + + - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_OTHER_ARTIFACTS + - run: *SUBMIT_COVERAGE diff --git a/.circleci/run-build-locally.sh b/.circleci/run-build-locally.sh new file mode 100755 index 000000000..32b801277 --- /dev/null +++ b/.circleci/run-build-locally.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CIRCLE_TOKEN=efb53124be82dd4b3153bc0e3f60de71da629d59 + +curl --user ${CIRCLE_TOKEN}: \ + --request POST \ + --form revision=$(git rev-parse HEAD) \ + --form config=@config.yml \ + --form notify=false \ + https://circleci.com/api/v1.1/project/github/exarkun/tahoe-lafs/tree/2929.circleci diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh new file mode 100755 index 000000000..25427e221 --- /dev/null +++ b/.circleci/run-tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash -e + +TAHOE_LAFS_TOX_ENVIRONMENT=$1 +shift + +TAHOE_LAFS_TOX_ARGS=$1 +shift || : + +# Run the test suite as a non-root user. This is the expected usage some +# small areas of the test suite assume non-root privileges (such as unreadable +# files being unreadable). +# +# Also run with /tmp as a workdir because the non-root user won't be able to +# create the tox working filesystem state in the source checkout because it is +# owned by root. +sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS} diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh new file mode 100755 index 000000000..690609777 --- /dev/null +++ b/.circleci/setup-virtualenv.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +TAHOE_LAFS_TOX_ENVIRONMENT=$1 +shift + +TAHOE_LAFS_TOX_ARGS=$1 +shift || : + +# Set up the virtualenv as a non-root user so we can run the test suite as a +# non-root user. See below. +sudo --set-home -u nobody virtualenv --python python2.7 /tmp/tests +sudo --set-home -u nobody /tmp/tests/bin/pip install tox codecov +# Get everything installed in it, too. +sudo --set-home -u nobody /tmp/tests/bin/tox -c /tmp/project/tox.ini --workdir /tmp --notest -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" ${TAHOE_LAFS_TOX_ARGS} diff --git a/README.rst b/README.rst index 072c016a9..3dfb9d4d8 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ function correctly, preserving your privacy and security. For full documentation, please see http://tahoe-lafs.readthedocs.io/en/latest/ . -|readthedocs| |travis| |codecov| +|readthedocs| |travis| |circleci| |codecov| INSTALLING ========== @@ -99,6 +99,9 @@ slides. :alt: build status :target: https://travis-ci.org/tahoe-lafs/tahoe-lafs +.. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg + :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs + .. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 774d73792..ac76e1d51 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -643,9 +643,31 @@ class FileUtil(ReallyEqualMixin, unittest.TestCase): self.failIfEqual(new_mode, 0766) def test_create_long_path(self): + """ + Even for paths with total length greater than 260 bytes, + ``fileutil.abspath_expanduser_unicode`` produces a path on which other + path-related APIs can operate. + + https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + documents certain Windows-specific path length limitations this test + is specifically intended to demonstrate can be overcome. + """ workdir = u"test_create_long_path" fileutil.make_dirs(workdir) - long_path = fileutil.abspath_expanduser_unicode(os.path.join(workdir, u'x'*255)) + base_path = fileutil.abspath_expanduser_unicode(workdir) + base_length = len(base_path) + + # Construct a path /just/ long enough to exercise the important case. + # It would be nice if we could just use a seemingly globally valid + # long file name (the `x...` portion) here - for example, a name 255 + # bytes long- and a previous version of this test did just that. + # However, aufs imposes a 242 byte length limit on file names. Most + # other POSIX filesystems do allow names up to 255 bytes. It's not + # clear there's anything we can *do* about lower limits, though, and + # POSIX.1-2017 (and earlier) only requires that the maximum be at + # least 14 (!!!) bytes. + long_path = os.path.join(base_path, u'x' * (261 - base_length)) + def _cleanup(): fileutil.remove(long_path) self.addCleanup(_cleanup) diff --git a/tox.ini b/tox.ini index d5850390d..d27cd0637 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,6 @@ deps = incremental usedevelop = True extras = test commands = - pyflakes src static misc setup.py tahoe --version trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} @@ -37,7 +36,6 @@ commands = [testenv:coverage] # coverage (with --branch) takes about 65% longer to run commands = - pyflakes src static misc setup.py tahoe --version coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata} coverage xml @@ -49,15 +47,12 @@ commands = python misc/coding_tools/check-debugging.py python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py python misc/coding_tools/check-miscaptures.py - # note: check-interfaces.py imports everything, so it must be run - # from a populated virtualenv - python misc/coding_tools/check-interfaces.py [testenv:deprecations] setenv = PYTHONWARNINGS=default::DeprecationWarning commands = - python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} + python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} [testenv:upcoming-deprecations] setenv = @@ -67,7 +62,7 @@ deps = git+https://github.com/warner/foolscap commands = flogtool --version - python misc/build_helpers/run-deprecations.py --warnings=_trial_temp/deprecation-warnings.log trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} + python misc/build_helpers/run-deprecations.py --warnings={env:TAHOE_LAFS_WARNINGS_LOG:_trial_temp/deprecation-warnings.log} trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata} [testenv:checkmemory] commands =