From 0639f2c16c70b51b5e5a965aedaaa0c220830ec8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:35:43 -0500 Subject: [PATCH 01/35] Try to switch to modern Python 3 world. Temporarily switch image building to always happen. --- ...ckerfile.centos => Dockerfile.oraclelinux} | 2 +- .circleci/config.yml | 205 +++++++----------- 2 files changed, 74 insertions(+), 133 deletions(-) rename .circleci/{Dockerfile.centos => Dockerfile.oraclelinux} (97%) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.oraclelinux similarity index 97% rename from .circleci/Dockerfile.centos rename to .circleci/Dockerfile.oraclelinux index 9070d71d9..ee31643b4 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.oraclelinux @@ -1,5 +1,5 @@ ARG TAG -FROM centos:${TAG} +FROM oraclelinux:${TAG} ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse diff --git a/.circleci/config.yml b/.circleci/config.yml index daf985567..ac62d9ed9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,28 +15,20 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-9": - {} - "debian-10": + {} + - "debian-11": requires: - - "debian-9" + - "debian-10" - "ubuntu-20-04": {} - "ubuntu-18-04": requires: - "ubuntu-20-04" - - "ubuntu-16-04": - requires: - - "ubuntu-20-04" - - "fedora-29": - {} - - "fedora-28": - requires: - - "fedora-29" - - - "centos-8": + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": {} - "nixos": @@ -47,9 +39,9 @@ workflows: name: "NixOS 21.11" nixpkgs: "21.11" - # Test against PyPy 2.7 - - "pypy27-buster": - {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} # Test against Python 3: - "python37": @@ -74,7 +66,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-9" + - "debian-10" - "typechecks": {} @@ -85,13 +77,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide @@ -104,22 +96,19 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-9": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-16-04": + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-18-04": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-28": + - "build-image-fedora-35": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-29": - <<: *DOCKERHUB_CONTEXT - - "build-image-centos-8": - <<: *DOCKERHUB_CONTEXT - - "build-image-pypy27-buster": + - "build-image-oraclelinux-8": <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -150,7 +139,7 @@ jobs: lint: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -168,7 +157,7 @@ jobs: codechecks3: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:3" + image: "cimg/python:3.9" steps: - "checkout" @@ -186,7 +175,7 @@ jobs: pyinstaller: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -209,10 +198,10 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-9: &DEBIAN + debian-10: &DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:9-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -226,7 +215,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py27" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -299,24 +288,32 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" - pypy27-buster: + debian-11: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/pypy:buster-py2" + image: "tahoelafsci/debian:11-py3.9" user: "nobody" - environment: - <<: *UTF_8_ENVIRONMENT - # We don't do coverage since it makes PyPy far too slow: - TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" - # Since we didn't collect it, don't upload it. - UPLOAD_COVERAGE: "" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 + # pypy27-buster: + # <<: *DEBIAN + # docker: + # - <<: *DOCKERHUB_AUTH + # image: "tahoelafsci/pypy:buster-py2" + # user: "nobody" + # environment: + # <<: *UTF_8_ENVIRONMENT + # # We don't do coverage since it makes PyPy far too slow: + # TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" + # # Since we didn't collect it, don't upload it. + # UPLOAD_COVERAGE: "" c-locale: <<: *DEBIAN @@ -364,23 +361,6 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - - ubuntu-16-04: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:16.04-py2.7" - user: "nobody" - - - ubuntu-18-04: &UBUNTU_18_04 - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py2.7" - user: "nobody" - - python37: <<: *UBUNTU_18_04 docker: @@ -405,10 +385,10 @@ jobs: user: "nobody" - centos-8: &RHEL_DERIV + oracelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/centos:8-py2" + image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -427,20 +407,11 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE - - fedora-28: + fedora-35: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:28-py" - user: "nobody" - - - fedora-29: - <<: *RHEL_DERIV - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:29-py" + image: "tahoelafsci/fedora:35-py3.9" user: "nobody" nixos: @@ -554,7 +525,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -566,7 +537,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -589,8 +560,8 @@ jobs: image: "cimg/base:2022.01" environment: - DISTRO: "tahoelafsci/:foo-py2" - TAG: "tahoelafsci/distro:-py2" + DISTRO: "tahoelafsci/:foo-py3.9" + TAG: "tahoelafsci/distro:-py3.9" PYTHON_VERSION: "tahoelafsci/distro:tag-py Date: Mon, 14 Feb 2022 10:40:38 -0500 Subject: [PATCH 02/35] Fix some references. --- .circleci/config.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac62d9ed9..b178d945b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,10 +43,6 @@ workflows: #- "pypy27-buster": # {} - # Test against Python 3: - - "python37": - {} - # Other assorted tasks and configurations - "lint": {} @@ -361,8 +357,8 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - python37: - <<: *UBUNTU_18_04 + ubuntu-18-04: &UBUNTU_18_04 + <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:18.04-py3.7" From d4810ce5b865ddcf4cfe3138b3a401e4034a3c67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:42:58 -0500 Subject: [PATCH 03/35] Get rid of duplicate. --- .circleci/config.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b178d945b..c90ce16b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,15 +279,6 @@ jobs: /tmp/venv/bin/codecov fi - - debian-10: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py3.7" - user: "nobody" - - debian-11: <<: *DEBIAN docker: From f0a81e1095a4a1ea90ba819e4e7b99a0b57f3617 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:44:44 -0500 Subject: [PATCH 04/35] Fix typo. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c90ce16b7..0dbcc1314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,7 +372,7 @@ jobs: user: "nobody" - oracelinux-8: &RHEL_DERIV + oraclelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/oraclelinux:8-py3.8" From 5935d9977697c6ae1e5ffc4f6618084071b33c5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:45:56 -0500 Subject: [PATCH 05/35] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0dbcc1314..511cee87c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-python37-ubuntu: + build-image-ubuntu-18.04: <<: *BUILD_IMAGE environment: From 4e133eb759daafce648fce593248d7930effe235 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:46:55 -0500 Subject: [PATCH 06/35] Fix name. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 511cee87c..31df8d6a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,7 +591,7 @@ jobs: TAG: "11" PYTHON_VERSION: "3.9" - build-image-ubuntu-18.04: + build-image-ubuntu-18-04: <<: *BUILD_IMAGE environment: From 19a3d2acf7b6fc1871d277219724f6cdb23ff016 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:49:17 -0500 Subject: [PATCH 07/35] Fix some more. --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31df8d6a1..2b588fbb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -105,8 +105,6 @@ workflows: # Restore later as PyPy38 #- "build-image-pypy27-buster": # <<: *DOCKERHUB_CONTEXT - - "build-image-python37-ubuntu": - <<: *DOCKERHUB_CONTEXT jobs: @@ -609,7 +607,7 @@ jobs: PYTHON_VERSION: "3.9" - build-image-oracelinux: + build-image-oraclelinux-8: <<: *BUILD_IMAGE environment: From 6a17c07158d56c94f2a129509d326daff779be57 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:52:33 -0500 Subject: [PATCH 08/35] Drop unnecessary install. --- .circleci/Dockerfile.oraclelinux | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/Dockerfile.oraclelinux b/.circleci/Dockerfile.oraclelinux index ee31643b4..cf4c009d2 100644 --- a/.circleci/Dockerfile.oraclelinux +++ b/.circleci/Dockerfile.oraclelinux @@ -13,7 +13,6 @@ RUN yum install --assumeyes \ sudo \ make automake gcc gcc-c++ \ python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ From 13d23e3baffceab6d726ddad5a37e579f4217c01 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 10:57:44 -0500 Subject: [PATCH 09/35] The terminal is a lie. --- .circleci/Dockerfile.debian | 2 +- .circleci/Dockerfile.ubuntu | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 96c54736c..f12f19551 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,7 +1,7 @@ ARG TAG FROM debian:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..22689f0c1 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,7 +1,7 @@ ARG TAG FROM ubuntu:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. From 0928a7993a588298e723c98bbe5d216ad713306f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:02:25 -0500 Subject: [PATCH 10/35] Rip out Python 2. --- .github/workflows/ci.yml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f65890d37..a4d8daca5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,14 +38,11 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.8 - 3.9 include: # On macOS don't bother with 3.7-3.8, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -108,25 +105,6 @@ jobs: # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the # coveralls.io-supplied Python reporter, for this. - # - # It is coveralls-python 1.x that has maintained compatibility - # with Python 2, while coveralls-python 3.x is compatible with - # Python 3. Sadly we can't use them both in the same workflow. - # - # The two versions of coveralls-python are somewhat mutually - # incompatible. Mixing these two different versions when - # reporting coverage to coveralls.io will lead to grief, since - # they get job IDs in different fashion. If we use both - # versions of coveralls in the same workflow, the finalizing - # step will be able to mark only part of the jobs as done, and - # the other part will be left hanging, never marked as done: it - # does not matter if we make an API call or `coveralls --finish` - # to indicate that CI has finished running. - # - # So we try to use the newer coveralls-python that is available - # via Python 3 (which is present in GitHub Actions tool cache, - # even when we're running Python 2.7 tests) throughout this - # workflow. - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 @@ -179,13 +157,10 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 - 3.7 - 3.9 include: # On macOS don't bother with 3.7, just to get faster builds. - - os: macos-10.15 - python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -242,12 +217,7 @@ jobs: - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "Python 2 integration tests" - if: ${{ matrix.python-version == '2.7' }} - run: tox -e integration - - name: Run "Python 3 integration tests" - if: ${{ matrix.python-version != '2.7' }} run: tox -e integration3 - name: Upload eliot.log in case of failure @@ -267,7 +237,7 @@ jobs: - windows-latest - ubuntu-latest python-version: - - 2.7 + - 3.9 steps: From 34fe6a41edd3c4468bfd78969b14cfed5d890a8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:05:31 -0500 Subject: [PATCH 11/35] Fix Fedora package name. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2b588fbb7..cd26a5de8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -396,7 +396,7 @@ jobs: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:35-py3.9" + image: "tahoelafsci/fedora:35-py3" user: "nobody" nixos: @@ -621,7 +621,7 @@ jobs: environment: DISTRO: "fedora" TAG: "35" - PYTHON_VERSION: "3.9" + PYTHON_VERSION: "3" # build-image-pypy27-buster: # <<: *BUILD_IMAGE From cd33e1cfb384df56cc59a46258c781f708f901c7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:10:56 -0500 Subject: [PATCH 12/35] Rip out more Python 2 stuff. --- Makefile | 8 ++++---- docs/release-checklist.rst | 2 +- tox.ini | 15 +-------------- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 33a40df02..fa8351535 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ PYTHON=python export PYTHON PYFLAKES=flake8 export PYFLAKES -VIRTUAL_ENV=./.tox/py27 +VIRTUAL_ENV=./.tox/py37 SOURCES=src/allmydata static misc setup.py APPNAME=tahoe-lafs TEST_SUITE=allmydata @@ -33,9 +33,9 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks + tox --develop -e codechecks3 # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py37,pypy27' + tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -136,7 +136,7 @@ count-lines: # Here is a list of testing tools that can be run with 'python' from a # virtualenv in which Tahoe has been installed. There used to be Makefile # targets for each, but the exact path to a suitable python is now up to the -# developer. But as a hint, after running 'tox', ./.tox/py27/bin/python will +# developer. But as a hint, after running 'tox', ./.tox/py37/bin/python will # probably work. # src/allmydata/test/bench_dirnode.py diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 9588fd1a5..5697ab95f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py27,codechecks,docs,integration + - tox -e py37,codechecks3,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 34d555aa7..3125548f4 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ # the tox-gh-actions package. [gh-actions] python = - 2.7: py27-coverage,codechecks 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -17,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -110,18 +109,6 @@ commands = coverage report -# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 -# since that can give different results than flake8 on Python 3. -[testenv:codechecks] -basepython = python2.7 -setenv = - # If no positional arguments are given, try to run the checks on the - # entire codebase, including various pieces of supporting code. - DEFAULT_FILES=src integration static misc setup.py -commands = - flake8 {posargs:{env:DEFAULT_FILES}} - - [testenv:codechecks3] basepython = python3 deps = From 9428c5d45b14c432d0f369643c567cdc3a8f5e18 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:29:19 -0500 Subject: [PATCH 13/35] We can use modern PyInstaller. --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 3125548f4..e42958b28 100644 --- a/tox.ini +++ b/tox.ini @@ -218,9 +218,7 @@ extras = deps = {[testenv]deps} packaging - # PyInstaller 4.0 drops Python 2 support. When we finish porting to - # Python 3 we can reconsider this constraint. - pyinstaller < 4.0 + pyinstaller pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build From d54294a6ec039a0bdbb6aeac138eec29996b6e5e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:09 -0500 Subject: [PATCH 14/35] News files. --- newsfragments/3327.minor | 0 newsfragments/3873.incompat | 1 + 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3327.minor create mode 100644 newsfragments/3873.incompat diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat new file mode 100644 index 000000000..7e71b1504 --- /dev/null +++ b/newsfragments/3873.incompat @@ -0,0 +1 @@ +Dropped support for Python 2. \ No newline at end of file From 77e7f80a1a93bec0683588e5ce19eff2df638543 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:19 -0500 Subject: [PATCH 15/35] Try to update to Python 3. --- misc/build_helpers/run-deprecations.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index f99cf90aa..2ad335bd1 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -26,10 +26,10 @@ python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] CO class RunPP(protocol.ProcessProtocol): def outReceived(self, data): self.stdout.write(data) - sys.stdout.write(data) + sys.stdout.write(str(data, sys.stdout.encoding)) def errReceived(self, data): self.stderr.write(data) - sys.stderr.write(data) + sys.stderr.write(str(data, sys.stdout.encoding)) def processEnded(self, reason): signal = reason.value.signal rc = reason.value.exitCode @@ -100,17 +100,19 @@ def run_command(main): pp.stdout.seek(0) for line in pp.stdout.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) # includes newline pp.stderr.seek(0) for line in pp.stderr.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) if warnings: if config["warnings"]: - with open(config["warnings"], "wb") as f: + with open(config["warnings"], "w") as f: print("".join(warnings), file=f) print("ERROR: %d deprecation warnings found" % len(warnings)) sys.exit(1) From 0bcfc58c2279dac8c94293aa8c0ea7fa14d9ce31 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:30:24 -0500 Subject: [PATCH 16/35] Various version fixes. --- .circleci/config.yml | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd26a5de8..9050c30b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,8 +44,6 @@ workflows: # {} # Other assorted tasks and configurations - - "lint": - {} - "codechecks3": {} - "pyinstaller": @@ -130,24 +128,6 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: - docker: - - <<: *DOCKERHUB_AUTH - image: "cimg/python:3.9" - - steps: - - "checkout" - - - run: - name: "Install tox" - command: | - pip install --user tox - - - run: - name: "Static-ish code checks" - command: | - ~/.local/bin/tox -e codechecks - codechecks3: docker: - <<: *DOCKERHUB_AUTH @@ -368,7 +348,8 @@ jobs: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:20.04" user: "nobody" - + environment: + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV docker: @@ -376,7 +357,9 @@ jobs: image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" - environment: *UTF_8_ENVIRONMENT + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py38" # pip cannot install packages if the working directory is not readable. # We want to run a lot of steps as nobody instead of as root. From 3ccd051473f364cc37cbc1c94658cb55f2ac4320 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:40:03 -0500 Subject: [PATCH 17/35] Correct image. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9050c30b4..bf0cbd8b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -346,7 +346,7 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:20.04" + image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: TAHOE_LAFS_TOX_ENVIRONMENT: "py39" From d976524bb09debd3f396da2a7d0b54d3dbcff3f3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 14 Feb 2022 11:41:51 -0500 Subject: [PATCH 18/35] Let's see if this is necessary any more. --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index e42958b28..dcbcc3ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -211,9 +211,6 @@ commands = sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] -# We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) -# is broken when this feature is enabled. -install_command = python -m pip install --no-use-pep517 {opts} {packages} extras = deps = {[testenv]deps} From dfe7de54a25868f8cdb636f03371d991ce9a031c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 09:59:04 -0500 Subject: [PATCH 19/35] Upgrade some versions. --- tox.ini | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index dcbcc3ab6..3916b807c 100644 --- a/tox.ini +++ b/tox.ini @@ -34,12 +34,10 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - # - # For now these are versions that support Python 2. - pip==20.3.4 - setuptools==44.1.1 - wheel==0.36.2 - subunitreporter==19.3.2 + pip==22.0.3 + setuptools==60.9.1 + wheel==0.37.1 + subunitreporter==22.2.0 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is # guaranteed to cause things to break eventually as old certificates From 1fd8603673f6dc89fdd5a25f55fff5dfd82a46eb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:07:04 -0500 Subject: [PATCH 20/35] Use modern Docker version (with bugfixes for modern distributions). --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf0cbd8b4..ac054b7eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -535,6 +535,7 @@ jobs: steps: - "checkout" - "setup_remote_docker" + version: "20.10.11" - run: name: "Log in to Dockerhub" command: | From 4315f6a641893022b8c1c3d1c7e670cb56f9746c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:12:31 -0500 Subject: [PATCH 21/35] Run on Python 3. --- pyinstaller.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 875629c13..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -11,7 +11,10 @@ import struct import sys -if not hasattr(sys, 'real_prefix'): +try: + import allmydata + del allmydata +except ImportError: sys.exit("Please run inside a virtualenv with Tahoe-LAFS installed.") From 95c32ef2eec363b369a3439aa1eb39e7a6aefe7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:13:35 -0500 Subject: [PATCH 22/35] Fix syntax. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac054b7eb..afb82f79a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -534,7 +534,7 @@ jobs: steps: - "checkout" - - "setup_remote_docker" + - setup_remote_docker: version: "20.10.11" - run: name: "Log in to Dockerhub" From 7ea106a018a92f13ba73fce9c8ac9691315b4284 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:19:56 -0500 Subject: [PATCH 23/35] Switch back to building Docker images on a schedule. --- .circleci/config.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index afb82f79a..7a4be90ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,13 +71,13 @@ workflows: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From be2590f9b8b95fccc795e0df47a89df58e882cc9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:20:29 -0500 Subject: [PATCH 24/35] Python 2 is now unsupported. --- README.rst | 9 ++++----- setup.py | 38 ++++++++++---------------------------- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index 0b73b520e..317378fae 100644 --- a/README.rst +++ b/README.rst @@ -53,12 +53,11 @@ For more detailed instructions, read `Installing Tahoe-LAFS `__ to learn how to set up your first Tahoe-LAFS node. -🐍 Python 3 Support --------------------- +🐍 Python 2 +----------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. -System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. -Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3.7 or later is now required. +If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. 🤖 Issues diff --git a/setup.py b/setup.py index c38c0bfec..da4950d2e 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,7 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 21.7.0 ; python_version > '3.0'", + "foolscap >= 21.7.0", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -106,16 +105,10 @@ install_requires = [ # for 'tahoe invite' and 'tahoe join' "magic-wormhole >= 0.10.2", - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - "eliot ~= 1.7 ; python_version < '3.0'", - # On Python 3, we want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0 ; python_version > '3.0'", + # We want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0", - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - "pyrsistent < 0.17.0 ; python_version < '3.0'", - "pyrsistent ; python_version > '3.0'", + "pyrsistent", # A great way to define types of values. "attrs >= 18.2.0", @@ -135,14 +128,8 @@ install_requires = [ # Linux distribution detection: "distro >= 1.4.0", - # Backported configparser for Python 2: - "configparser ; python_version < '3.0'", - - # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python - # 2 doesn't actually need this, since HTTP storage protocol isn't supported - # there, so we just pick whatever version so that code imports. - "collections-extended >= 2.0.2 ; python_version > '3.0'", - "collections-extended ; python_version < '3.0'", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. + "collections-extended >= 2.0.2", # HTTP server and client "klein", @@ -201,8 +188,7 @@ trove_classifiers=[ "Natural Language :: English", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", "Topic :: Utilities", "Topic :: System :: Systems Administration", "Topic :: System :: Filesystems", @@ -229,7 +215,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -380,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and Python 3.7 or later. - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", + # We support Python 3.7 or later. + python_requires=">=3.7", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -400,10 +386,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - "decorator < 5", "hypothesis >= 3.6.1", "towncrier", "testtools", From bd907289468858d1a8e0d0a46a383ffb03516ce2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:26:54 -0500 Subject: [PATCH 25/35] Re-add missing environment. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a4be90ab..b1d73e89f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -264,6 +264,7 @@ jobs: image: "tahoelafsci/debian:11-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" # Restore later using PyPy3.8 @@ -349,6 +350,7 @@ jobs: image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" environment: + <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py39" oraclelinux-8: &RHEL_DERIV From 3255f93a5c1f94af75d05a2d07bf8851d74df369 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 10:47:22 -0500 Subject: [PATCH 26/35] Try newer version of Chutney. --- integration/conftest.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index ef5c518a8..e284b5cba 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -462,10 +462,8 @@ def chutney(reactor, temp_dir): ) pytest_twisted.blockon(proto.done) - # XXX: Here we reset Chutney to the last revision known to work - # with Python 2, as a workaround for Chutney moving to Python 3. - # When this is no longer necessary, we will have to drop this and - # add '--depth=1' back to the above 'git clone' subprocess. + # XXX: Here we reset Chutney to a specific revision known to work, + # since there are no stability guarantees or releases yet. proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -473,7 +471,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - '99bd06c7554b9113af8c0877b6eca4ceb95dcbaa' + 'c825cba0bcd813c644c6ac069deeb7347d3200ee' ), env=environ, ) From 6190399aef2f61f318bb5a0c78e5a1e9f8a7e335 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:00 -0500 Subject: [PATCH 27/35] Just codechecks. --- .circleci/config.yml | 6 +++--- Makefile | 2 +- docs/release-checklist.rst | 2 +- tox.ini | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1d73e89f..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ workflows: # {} # Other assorted tasks and configurations - - "codechecks3": + - "codechecks": {} - "pyinstaller": {} @@ -128,7 +128,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - codechecks3: + codechecks: docker: - <<: *DOCKERHUB_AUTH image: "cimg/python:3.9" @@ -144,7 +144,7 @@ jobs: - run: name: "Static-ish code checks" command: | - ~/.local/bin/tox -e codechecks3 + ~/.local/bin/tox -e codechecks pyinstaller: docker: diff --git a/Makefile b/Makefile index fa8351535..5cbd863a3 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ default: ## Run all tests and code reports test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. - tox --develop -e codechecks3 + tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time tox --develop -p auto -e 'py37' .PHONY: test-venv-coverage diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 5697ab95f..aa5531b59 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py37,codechecks3,docs,integration + - tox -e py37,codechecks,docs,integration - these can fail (ideally they should not of course): diff --git a/tox.ini b/tox.ini index 3916b807c..525b5428c 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 3.7: py37-coverage,typechecks,codechecks3 + 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage pypy-3.7: pypy3 @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks3,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -107,7 +107,7 @@ commands = coverage report -[testenv:codechecks3] +[testenv:codechecks] basepython = python3 deps = # Newer versions of PyLint have buggy configuration From b7b71b2de7a6526564f5d509e29d6168ae1eb776 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:33:37 -0500 Subject: [PATCH 28/35] Clarify. --- newsfragments/3873.incompat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat index 7e71b1504..da8a5fb0e 100644 --- a/newsfragments/3873.incompat +++ b/newsfragments/3873.incompat @@ -1 +1 @@ -Dropped support for Python 2. \ No newline at end of file +Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file From 21e288a4d0e4b9078edd17589fd72eeda4f5a079 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:35:18 -0500 Subject: [PATCH 29/35] Technically don't support 3.10 yet. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da4950d2e..8bb2b57aa 100644 --- a/setup.py +++ b/setup.py @@ -366,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 3.7 or later. - python_requires=">=3.7", + # We support Python 3.7 or later. 3.10 is not supported quite yet. + python_requires=">=3.7, <3.10", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See From 510102dab1f6a5b38a14bffd789bfd5188c8b6fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:37:18 -0500 Subject: [PATCH 30/35] Maybe we can use modern tor now. --- .github/workflows/ci.yml | 7 +------ newsfragments/3744.minor | 0 2 files changed, 1 insertion(+), 6 deletions(-) create mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4d8daca5..a0f7889b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,15 +170,10 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. - # We have to use an older version of Tor for running integration - # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.5.8 tor homebrew/cask - brew install tor@0.4.5.8 - brew link --overwrite tor@0.4.5.8 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor new file mode 100644 index 000000000..e69de29bb From 3a859e3cac580370682c5124744dd486d580600c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 15 Feb 2022 14:51:55 -0500 Subject: [PATCH 31/35] Try a version that matches Ubuntu's. --- .github/workflows/ci.yml | 7 ++++++- newsfragments/3744.minor | 0 2 files changed, 6 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/3744.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0f7889b5..8e6b0fd67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,10 +170,15 @@ jobs: if: matrix.os == 'ubuntu-latest' run: sudo apt install tor + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. + # We have to use an older version of Tor for running integration + # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew install tor + brew extract --version 0.4.2.7 tor homebrew/cask + brew install tor@0.4.2.7 + brew link --overwrite tor@0.4.2.7 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' diff --git a/newsfragments/3744.minor b/newsfragments/3744.minor deleted file mode 100644 index e69de29bb..000000000 From 84094b5ca00bd1af8168030f48678748d9ba4faa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Feb 2022 09:31:12 -0500 Subject: [PATCH 32/35] Try version that Windows uses. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e6b0fd67..3a12f51f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,9 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.2.7 tor homebrew/cask - brew install tor@0.4.2.7 - brew link --overwrite tor@0.4.2.7 + brew extract --version 0.4.6.9 tor homebrew/cask + brew install tor@0.4.6.9 + brew link --overwrite tor@0.4.6.9 - name: Install Tor [Windows] if: matrix.os == 'windows-latest' From a7e4add602b390f2e7f482725182ce8e62475ac2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:25:13 -0500 Subject: [PATCH 33/35] Simplify. --- .github/workflows/ci.yml | 6 ++---- tox.ini | 15 +++------------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a12f51f2..5f33dcd96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,9 +176,7 @@ jobs: - name: Install Tor [macOS, ${{ matrix.python-version }} ] if: ${{ contains(matrix.os, 'macos') }} run: | - brew extract --version 0.4.6.9 tor homebrew/cask - brew install tor@0.4.6.9 - brew link --overwrite tor@0.4.6.9 + brew install tor - name: Install Tor [Windows] if: matrix.os == 'windows-latest' @@ -218,7 +216,7 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" - run: tox -e integration3 + run: tox -e integration - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/tox.ini b/tox.ini index 525b5428c..52c199d41 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration minversion = 2.4 [testenv] @@ -86,21 +86,12 @@ commands = coverage: coverage report [testenv:integration] -setenv = - COVERAGE_PROCESS_START=.coveragerc -commands = - # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} - coverage combine - coverage report - - -[testenv:integration3] basepython = python3 setenv = COVERAGE_PROCESS_START=.coveragerc + # Without this, temporary file paths are too long on macOS, breaking tor + commands = - python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine From 52f0e18d6bfe2cc7e5e53d1b6cd77ecc2bb78e7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:26:00 -0500 Subject: [PATCH 34/35] Fix for overly-long temporary paths for unix sockets on macOS. --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 52c199d41..c0bc159da 100644 --- a/tox.ini +++ b/tox.ini @@ -87,10 +87,13 @@ commands = [testenv:integration] basepython = python3 +platform = mylinux: linux + mymacos: darwin + mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc # Without this, temporary file paths are too long on macOS, breaking tor - + mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} From 5647b4aee0ce88b8e8761588453dc608f0307d25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Feb 2022 11:40:16 -0500 Subject: [PATCH 35/35] Try to fix macOS another way. --- .github/workflows/ci.yml | 5 +++++ tox.ini | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f33dcd96..90778c6ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -216,6 +216,11 @@ jobs: run: python misc/build_helpers/show-tool-versions.py - name: Run "Python 3 integration tests" + env: + # On macOS this is necessary to ensure unix socket paths for tor + # aren't too long. On Windows tox won't pass it through so it has no + # effect. On Linux it doesn't make a difference one way or another. + TMPDIR: "/tmp" run: tox -e integration - name: Upload eliot.log in case of failure diff --git a/tox.ini b/tox.ini index c0bc159da..9a28b7b30 100644 --- a/tox.ini +++ b/tox.ini @@ -92,8 +92,6 @@ platform = mylinux: linux mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc - # Without this, temporary file paths are too long on macOS, breaking tor - mymacos: TMPDIR=/tmp commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration}