diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh index 810ce5ae2..7327d0859 100755 --- a/.circleci/create-virtualenv.sh +++ b/.circleci/create-virtualenv.sh @@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" # above, it may still not be able to get us a compatible version unless we # explicitly ask for one. "${PIP}" install --upgrade setuptools==44.0.0 wheel + +# Just about every user of this image wants to use tox from the bootstrap +# virtualenv so go ahead and install it now. +"${PIP}" install "tox~=3.0" diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 374ca0adb..239c8367b 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -3,18 +3,6 @@ # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ set -euxo pipefail -# Basic Python packages that you just need to have around to do anything, -# practically speaking. -BASIC_DEPS="pip wheel" - -# Python packages we need to support the test infrastructure. *Not* packages -# Tahoe-LAFS itself (implementation or test suite) need. -TEST_DEPS="tox~=3.0" - -# Python packages we need to generate test reports for CI infrastructure. -# *Not* packages Tahoe-LAFS itself (implement or test suite) need. -REPORTING_DEPS="python-subunit junitxml subunitreporter" - # The filesystem location of the wheelhouse which we'll populate with wheels # for all of our dependencies. WHEELHOUSE_PATH="$1" @@ -41,15 +29,5 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ - "${PROJECT_ROOT}"[test] \ - ${BASIC_DEPS} \ - ${TEST_DEPS} \ - ${REPORTING_DEPS} - -# Not strictly wheelhouse population but ... Note we omit basic deps here. -# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will -# have to ask. -"${PIP}" \ - install \ - ${TEST_DEPS} \ - ${REPORTING_DEPS} + "${PROJECT_ROOT}"[testenv] \ + "${PROJECT_ROOT}"[test] diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 6d7a881fe..d897cc729 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -79,9 +79,10 @@ else alternative="false" fi +WORKDIR=/tmp/tahoe-lafs.tox ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ -c ${PROJECT_ROOT}/tox.ini \ - --workdir /tmp/tahoe-lafs.tox \ + --workdir "${WORKDIR}" \ -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \ ${TAHOE_LAFS_TOX_ARGS} || "${alternative}" @@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" + + "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" fi diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh index feccbbf23..7087c5120 100755 --- a/.circleci/setup-virtualenv.sh +++ b/.circleci/setup-virtualenv.sh @@ -26,12 +26,7 @@ shift || : # Tell pip where it can find any existing wheels. export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" - -# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems -# between the time dependencies change and the images are re-built and (b) the -# upcoming-deprecations job wants to install some dependencies from github and -# it's awkward to get that done any earlier than the tox run. So, we don't -# set it. +export PIP_NO_INDEX="1" # Get everything else installed in it, too. "${BOOTSTRAP_VENV}"/bin/tox \ diff --git a/newsfragments/4019.minor b/newsfragments/4019.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/4020.minor b/newsfragments/4020.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/4020.minor @@ -0,0 +1 @@ + diff --git a/setup.py b/setup.py index 2418c6dbe..4f28b4438 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,10 @@ install_requires = [ # HTTP server and client "klein", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 - "werkzeug != 2.2.0", + # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575 + "werkzeug != 2.2.0, < 2.3", "treq", "cbor2", @@ -398,10 +400,31 @@ setup(name="tahoe-lafs", # also set in __init__.py "dulwich", "gpg", ], - "test": [ + + # Here are the dependencies required to set up a reproducible test + # environment. This could be for CI or local development. These + # are *not* library dependencies of the test suite itself. They are + # the tools we use to run the test suite at all. + "testenv": [ + # Pin all of these versions for the same reason you ever want to + # pin anything: to prevent new releases with regressions from + # introducing spurious failures into CI runs for whatever + # development work is happening at the time. The versions + # selected here are just the current versions at the time. + # Bumping them to keep up with future releases is fine as long + # as those releases are known to actually work. + "pip==22.0.3", + "wheel==0.37.1", + "setuptools==60.9.1", + "subunitreporter==22.2.0", + "python-subunit==1.4.2", + "junitxml==0.7", "coverage ~= 5.0", + ], + + # Here are the library dependencies of the test suite. + "test": [ "mock", - "tox ~= 3.0", "pytest", "pytest-twisted", "hypothesis >= 3.6.1", @@ -410,7 +433,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "fixtures", "beautifulsoup4", "html5lib", - "junitxml", # Pin old version until # https://github.com/paramiko/paramiko/issues/1961 is fixed. "paramiko < 2.9", diff --git a/tox.ini b/tox.ini index 5f18b6b95..2edb15a0b 100644 --- a/tox.ini +++ b/tox.ini @@ -23,38 +23,34 @@ minversion = 2.4 [testenv] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH -# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes -# a package to be installed (with setuptools) then it'll fail on certain -# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS -# requirements (TLS >= 1.2) are incompatible with the old TLS clients -# available to those systems. Installing it ahead of time (with pip) avoids -# this problem. deps = - # Pin all of these versions for the same reason you ever want to pin - # anything: to prevent new releases with regressions from introducing - # spurious failures into CI runs for whatever development work is - # happening at the time. The versions selected here are just the current - # versions at the time. Bumping them to keep up with future releases is - # fine as long as those releases are known to actually work. - pip==22.0.3 - setuptools==60.9.1 - wheel==0.37.1 - subunitreporter==22.2.0 - # As an exception, we don't pin certifi because it contains CA - # certificates which necessarily change over time. Pinning this is - # guaranteed to cause things to break eventually as old certificates - # expire and as new ones are used in the wild that aren't present in - # whatever version we pin. Hopefully there won't be functionality - # regressions in new releases of this package that cause us the kind of - # suffering we're trying to avoid with the above pins. - certifi + # We pull in certify *here* to avoid bug #2913. Basically if a + # `setup_requires=...` causes a package to be installed (with setuptools) + # then it'll fail on certain platforms (travis's OX-X 10.12, Slackware + # 14.2) because PyPI's TLS requirements (TLS >= 1.2) are incompatible with + # the old TLS clients available to those systems. Installing it ahead of + # time (with pip) avoids this problem. + # + # We don't pin an exact version of it because it contains CA certificates + # which necessarily change over time. Pinning this is guaranteed to cause + # things to break eventually as old certificates expire and as new ones + # are used in the wild that aren't present in whatever version we pin. + # Hopefully there won't be functionality regressions in new releases of + # this package that cause us the kind of suffering we're trying to avoid + # with the above pins. + certifi # We add usedevelop=False because testing against a true installation gives # more useful results. usedevelop = False -# We use extras=test to get things like "mock" that are required for our unit -# tests. -extras = test + +extras = + # Get general testing environment dependencies so we can run the tests + # how we like. + testenv + + # And get all of the test suite's actual direct Python dependencies. + test setenv = # Define TEST_SUITE in the environment as an aid to constructing the