From 764e493c98798f2b4248189e1c0faa3a9df27ffb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:32:27 -0500 Subject: [PATCH 1/9] News. --- newsfragments/3865.incompat | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3865.incompat diff --git a/newsfragments/3865.incompat b/newsfragments/3865.incompat new file mode 100644 index 000000000..59381b269 --- /dev/null +++ b/newsfragments/3865.incompat @@ -0,0 +1 @@ +Python 3.6 is no longer supported, as it has reached end-of-life and is no longer receiving security updates. \ No newline at end of file From 8eb6ab47653f7e8be52c310332f8f6d9686cd2ae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:40:26 -0500 Subject: [PATCH 2/9] Switch to Python 3.7 as minimal version. --- .circleci/config.yml | 12 ++++++------ .github/workflows/ci.yml | 7 +++---- Makefile | 4 ++-- misc/python3/Makefile | 4 ++-- setup.py | 5 ++--- tox.ini | 5 ++--- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc8e88e7..d55b80469 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,8 +49,8 @@ workflows: - "pypy27-buster": {} - # Just one Python 3.6 configuration while the port is in-progress. - - "python36": + # Test against Python 3: + - "python37": {} # Other assorted tasks and configurations @@ -118,7 +118,7 @@ workflows: <<: *DOCKERHUB_CONTEXT - "build-image-pypy27-buster": <<: *DOCKERHUB_CONTEXT - - "build-image-python36-ubuntu": + - "build-image-python37-ubuntu": <<: *DOCKERHUB_CONTEXT @@ -379,7 +379,7 @@ jobs: user: "nobody" - python36: + python37: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH @@ -392,7 +392,7 @@ jobs: # this reporter on Python 3. So drop that and just specify the # reporter. TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file" - TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" ubuntu-20-04: @@ -577,7 +577,7 @@ jobs: PYTHON_VERSION: "2.7" - build-image-python36-ubuntu: + build-image-python37-ubuntu: <<: *BUILD_IMAGE environment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8209108bf..5ae70a3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,12 +39,11 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 - 3.7 - 3.8 - 3.9 include: - # On macOS don't bother with 3.6-3.8, just to get faster builds. + # On macOS don't bother with 3.7-3.8, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest @@ -181,10 +180,10 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + - 3.7 - 3.9 include: - # On macOS don't bother with 3.6, just to get faster builds. + # On macOS don't bother with 3.7, just to get faster builds. - os: macos-10.15 python-version: 2.7 - os: macos-latest diff --git a/Makefile b/Makefile index 5d8bf18ba..33a40df02 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ test: .tox/create-venvs.log # Run codechecks first since it takes the least time to report issues early. tox --develop -e codechecks # Run all the test environments in parallel to reduce run-time - tox --develop -p auto -e 'py27,py36,pypy27' + tox --develop -p auto -e 'py27,py37,pypy27' .PHONY: test-venv-coverage ## Run all tests with coverage collection and reporting. test-venv-coverage: @@ -51,7 +51,7 @@ test-venv-coverage: .PHONY: test-py3-all ## Run all tests under Python 3 test-py3-all: .tox/create-venvs.log - tox --develop -e py36 allmydata + tox --develop -e py37 allmydata # This is necessary only if you want to automatically produce a new # _version.py file from the current git history (without doing a build). diff --git a/misc/python3/Makefile b/misc/python3/Makefile index f0ef8b12a..43cb3e3ce 100644 --- a/misc/python3/Makefile +++ b/misc/python3/Makefile @@ -37,8 +37,8 @@ test-py3-all-diff: ../../.tox/make-test-py3-all.diff # `$ make .tox/make-test-py3-all.diff` $(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): cd "../../" - tox --develop --notest -e py36-coverage - (make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \ + tox --develop --notest -e py37-coverage + (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ test-venv-coverage || true) | \ sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ tee "./misc/python3/$(@)" diff --git a/setup.py b/setup.py index 53057b808..9a1c76bd8 100644 --- a/setup.py +++ b/setup.py @@ -376,9 +376,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and we're working on support for 3.6 (the - # highest version that PyPy currently supports). - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + # We support Python 2.7, and Python 3.7 or later. + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See diff --git a/tox.ini b/tox.ini index 38cee1f9f..34d555aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ [gh-actions] python = 2.7: py27-coverage,codechecks - 3.6: py36-coverage 3.7: py37-coverage,typechecks,codechecks3 3.8: py38-coverage 3.9: py39-coverage @@ -18,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,36,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 minversion = 2.4 [testenv] @@ -51,7 +50,7 @@ deps = # suffering we're trying to avoid with the above pins. certifi # VCS hooks support - py36,!coverage: pre-commit + py37,!coverage: pre-commit # We add usedevelop=False because testing against a true installation gives # more useful results. From fa2b4a11c76a1b55100ea51fa25f9d0dcda9ff6d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:50:40 -0500 Subject: [PATCH 3/9] Welcome to the WORLD OF TOMORROW --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d55b80469..5ef9c81a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,7 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:17.05.0-ce-git" + image: "docker:20.10" environment: DISTRO: "tahoelafsci/:foo-py2" From f04e121a7d31d1b18bdd5c5d40618f98ce3bb2ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:51:55 -0500 Subject: [PATCH 4/9] Try to use correct Docker image. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ef9c81a8..05686fcf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -383,7 +383,7 @@ jobs: <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" user: "nobody" environment: @@ -583,7 +583,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" - PYTHON_VERSION: "3" + PYTHON_VERSION: "3.7" build-image-ubuntu-20-04: From 02740f075bb69f74ae3ed198a426a0fb3750a5e1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 10:56:11 -0500 Subject: [PATCH 5/9] Temporarily enable image builds on every push. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 05686fcf8..6578f1d1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 31e4556bd1910c19a74e30b793d5098ca5f27b8c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:01:47 -0500 Subject: [PATCH 6/9] Need image with Docker _and_ git+ssh. --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6578f1d1a..d28196097 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -511,7 +511,9 @@ jobs: # https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/ docker: - <<: *DOCKERHUB_AUTH - image: "docker:20.10" + # CircleCI build images; https://github.com/CircleCI-Public/cimg-base + # for details. + image: "cimg/base:2022.01" environment: DISTRO: "tahoelafsci/:foo-py2" From 04cf206e0d78a174c0f80a80180340838f332b67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Jan 2022 11:06:58 -0500 Subject: [PATCH 7/9] Switch back to running image building on schedule. --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d28196097..a650313ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,12 +84,12 @@ workflows: # faster and takes various spurious failures out of the critical path. triggers: # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" jobs: # Every job that pushes a Docker image from Docker Hub needs to provide From 54996185dec11b7b2c78a1f90e6a17bee9ff3ed6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:05 -0500 Subject: [PATCH 8/9] No longer used. --- misc/python3/Makefile | 53 ------------------------------------------- 1 file changed, 53 deletions(-) delete mode 100644 misc/python3/Makefile diff --git a/misc/python3/Makefile b/misc/python3/Makefile deleted file mode 100644 index 43cb3e3ce..000000000 --- a/misc/python3/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Python 3 porting targets -# -# NOTE: this Makefile requires GNU make - -### Defensive settings for make: -# https://tech.davis-hansson.com/p/make/ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -xeu -o pipefail -c -.SILENT: -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules - - -# Top-level, phony targets - -.PHONY: default -default: - @echo "no default target" - -.PHONY: test-py3-all-before -## Log the output of running all tests under Python 3 before changes -test-py3-all-before: ../../.tox/make-test-py3-all-old.log -.PHONY: test-py3-all-diff -## Compare the output of running all tests under Python 3 after changes -test-py3-all-diff: ../../.tox/make-test-py3-all.diff - - -# Real targets - -# Gauge the impact of changes on Python 3 compatibility -# Compare the output from running all tests under Python 3 before and after changes. -# Before changes: -# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log` -# After changes: -# `$ make .tox/make-test-py3-all.diff` -$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log): - cd "../../" - tox --develop --notest -e py37-coverage - (make VIRTUAL_ENV=./.tox/py37-coverage TEST_SUITE=allmydata \ - test-venv-coverage || true) | \ - sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \ - tee "./misc/python3/$(@)" -../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log - (diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)" - -# Locate modules that are candidates for naively converting `unicode` -> `str`. -# List all Python source files that reference `unicode` but don't reference `str` -../../.tox/py3-unicode-no-str.ls: - cd "../../" - find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \ - xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)" From e1f9f7de94c68d8c4584ea650e6d9584614b3eb7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jan 2022 10:06:18 -0500 Subject: [PATCH 9/9] Note for future improvement. --- src/allmydata/util/encodingutil.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index f32710688..5e28f59fe 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -320,6 +320,9 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): # Although the problem is that doesn't work in Python 3.6, only 3.7 or # later... For now not thinking about it, just returning unicode since # that is the right thing to do on Python 3. + # + # Now that Python 3.7 is the minimum, this can in theory be done: + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3866 result = result.decode(encoding) return result