diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 96c54736c..f12f19551 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,7 +1,7 @@ ARG TAG FROM debian:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.oraclelinux similarity index 93% rename from .circleci/Dockerfile.centos rename to .circleci/Dockerfile.oraclelinux index 9070d71d9..cf4c009d2 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.oraclelinux @@ -1,5 +1,5 @@ ARG TAG -FROM centos:${TAG} +FROM oraclelinux:${TAG} ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -13,7 +13,6 @@ RUN yum install --assumeyes \ sudo \ make automake gcc gcc-c++ \ python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..22689f0c1 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,7 +1,7 @@ ARG TAG FROM ubuntu:${TAG} ARG PYTHON_VERSION - +ENV DEBIAN_FRONTEND noninteractive ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv # This will get updated by the CircleCI checkout step. diff --git a/.circleci/config.yml b/.circleci/config.yml index daf985567..cf0c66aff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,28 +15,20 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - "debian-9": - {} - "debian-10": + {} + - "debian-11": requires: - - "debian-9" + - "debian-10" - "ubuntu-20-04": {} - "ubuntu-18-04": requires: - "ubuntu-20-04" - - "ubuntu-16-04": - requires: - - "ubuntu-20-04" - - "fedora-29": - {} - - "fedora-28": - requires: - - "fedora-29" - - - "centos-8": + # Equivalent to RHEL 8; CentOS 8 is dead. + - "oraclelinux-8": {} - "nixos": @@ -47,18 +39,12 @@ workflows: name: "NixOS 21.11" nixpkgs: "21.11" - # Test against PyPy 2.7 - - "pypy27-buster": - {} - - # Test against Python 3: - - "python37": - {} + # Eventually, test against PyPy 3.8 + #- "pypy27-buster": + # {} # Other assorted tasks and configurations - - "lint": - {} - - "codechecks3": + - "codechecks": {} - "pyinstaller": {} @@ -74,7 +60,7 @@ workflows: requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - - "debian-9" + - "debian-10" - "typechecks": {} @@ -104,24 +90,19 @@ workflows: # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "build-image-debian-10": &DOCKERHUB_CONTEXT context: "dockerhub-auth" - - "build-image-debian-9": - <<: *DOCKERHUB_CONTEXT - - "build-image-ubuntu-16-04": + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-18-04": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-28": + - "build-image-fedora-35": <<: *DOCKERHUB_CONTEXT - - "build-image-fedora-29": - <<: *DOCKERHUB_CONTEXT - - "build-image-centos-8": - <<: *DOCKERHUB_CONTEXT - - "build-image-pypy27-buster": - <<: *DOCKERHUB_CONTEXT - - "build-image-python37-ubuntu": + - "build-image-oraclelinux-8": <<: *DOCKERHUB_CONTEXT + # Restore later as PyPy38 + #- "build-image-pypy27-buster": + # <<: *DOCKERHUB_CONTEXT jobs: @@ -147,10 +128,10 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: + codechecks: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -165,28 +146,10 @@ jobs: command: | ~/.local/bin/tox -e codechecks - codechecks3: - docker: - - <<: *DOCKERHUB_AUTH - image: "circleci/python:3" - - steps: - - "checkout" - - - run: - name: "Install tox" - command: | - pip install --user tox - - - run: - name: "Static-ish code checks" - command: | - ~/.local/bin/tox -e codechecks3 - pyinstaller: docker: - <<: *DOCKERHUB_AUTH - image: "circleci/python:2" + image: "cimg/python:3.9" steps: - "checkout" @@ -209,10 +172,10 @@ jobs: command: | dist/Tahoe-LAFS/tahoe --version - debian-9: &DEBIAN + debian-10: &DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:9-py2.7" + image: "tahoelafsci/debian:10-py3.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -226,7 +189,7 @@ jobs: # filenames and argv). LANG: "en_US.UTF-8" # Select a tox environment to run for this job. - TAHOE_LAFS_TOX_ENVIRONMENT: "py27" + TAHOE_LAFS_TOX_ENVIRONMENT: "py37" # Additional arguments to pass to tox. TAHOE_LAFS_TOX_ARGS: "" # The path in which test artifacts will be placed. @@ -294,29 +257,29 @@ jobs: /tmp/venv/bin/codecov fi - - debian-10: + debian-11: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/debian:10-py2.7" + image: "tahoelafsci/debian:11-py3.9" user: "nobody" - - - pypy27-buster: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/pypy:buster-py2" - user: "nobody" - environment: <<: *UTF_8_ENVIRONMENT - # We don't do coverage since it makes PyPy far too slow: - TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" - # Since we didn't collect it, don't upload it. - UPLOAD_COVERAGE: "" + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" + # Restore later using PyPy3.8 + # pypy27-buster: + # <<: *DEBIAN + # docker: + # - <<: *DOCKERHUB_AUTH + # image: "tahoelafsci/pypy:buster-py2" + # user: "nobody" + # environment: + # <<: *UTF_8_ENVIRONMENT + # # We don't do coverage since it makes PyPy far too slow: + # TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27" + # # Since we didn't collect it, don't upload it. + # UPLOAD_COVERAGE: "" c-locale: <<: *DEBIAN @@ -364,25 +327,8 @@ jobs: - run: *SETUP_VIRTUALENV - run: *RUN_TESTS - - ubuntu-16-04: - <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:16.04-py2.7" - user: "nobody" - - ubuntu-18-04: &UBUNTU_18_04 <<: *DEBIAN - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py2.7" - user: "nobody" - - - python37: - <<: *UBUNTU_18_04 docker: - <<: *DOCKERHUB_AUTH image: "tahoelafsci/ubuntu:18.04-py3.7" @@ -401,17 +347,21 @@ jobs: <<: *DEBIAN docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:20.04" + image: "tahoelafsci/ubuntu:20.04-py3.9" user: "nobody" + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py39" - - centos-8: &RHEL_DERIV + oraclelinux-8: &RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/centos:8-py2" + image: "tahoelafsci/oraclelinux:8-py3.8" user: "nobody" - environment: *UTF_8_ENVIRONMENT + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py38" # pip cannot install packages if the working directory is not readable. # We want to run a lot of steps as nobody instead of as root. @@ -427,20 +377,11 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE - - fedora-28: + fedora-35: <<: *RHEL_DERIV docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:28-py" - user: "nobody" - - - fedora-29: - <<: *RHEL_DERIV - docker: - - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/fedora:29-py" + image: "tahoelafsci/fedora:35-py3" user: "nobody" nixos: @@ -554,7 +495,7 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -566,7 +507,7 @@ jobs: docs: docker: - <<: *DOCKERHUB_AUTH - image: "tahoelafsci/ubuntu:18.04-py3" + image: "tahoelafsci/ubuntu:18.04-py3.7" steps: - "checkout" @@ -589,13 +530,14 @@ jobs: image: "cimg/base:2022.01" environment: - DISTRO: "tahoelafsci/:foo-py2" - TAG: "tahoelafsci/distro:-py2" + DISTRO: "tahoelafsci/:foo-py3.9" + TAG: "tahoelafsci/distro:-py3.9" PYTHON_VERSION: "tahoelafsci/distro:tag-py`__ to learn how to set up your first Tahoe-LAFS node. -🐍 Python 3 Support --------------------- +🐍 Python 2 +----------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. -System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. -Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3.7 or later is now required. +If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1. 🤖 Issues diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index 9588fd1a5..aa5531b59 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -122,7 +122,7 @@ they will need to evaluate which contributors' signatures they trust. - these should all pass: - - tox -e py27,codechecks,docs,integration + - tox -e py37,codechecks,docs,integration - these can fail (ideally they should not of course): diff --git a/integration/conftest.py b/integration/conftest.py index ef5c518a8..e284b5cba 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -462,10 +462,8 @@ def chutney(reactor, temp_dir): ) pytest_twisted.blockon(proto.done) - # XXX: Here we reset Chutney to the last revision known to work - # with Python 2, as a workaround for Chutney moving to Python 3. - # When this is no longer necessary, we will have to drop this and - # add '--depth=1' back to the above 'git clone' subprocess. + # XXX: Here we reset Chutney to a specific revision known to work, + # since there are no stability guarantees or releases yet. proto = _DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -473,7 +471,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - '99bd06c7554b9113af8c0877b6eca4ceb95dcbaa' + 'c825cba0bcd813c644c6ac069deeb7347d3200ee' ), env=environ, ) diff --git a/misc/build_helpers/run-deprecations.py b/misc/build_helpers/run-deprecations.py index f99cf90aa..2ad335bd1 100644 --- a/misc/build_helpers/run-deprecations.py +++ b/misc/build_helpers/run-deprecations.py @@ -26,10 +26,10 @@ python run-deprecations.py [--warnings=STDERRFILE] [--package=PYTHONPACKAGE ] CO class RunPP(protocol.ProcessProtocol): def outReceived(self, data): self.stdout.write(data) - sys.stdout.write(data) + sys.stdout.write(str(data, sys.stdout.encoding)) def errReceived(self, data): self.stderr.write(data) - sys.stderr.write(data) + sys.stderr.write(str(data, sys.stdout.encoding)) def processEnded(self, reason): signal = reason.value.signal rc = reason.value.exitCode @@ -100,17 +100,19 @@ def run_command(main): pp.stdout.seek(0) for line in pp.stdout.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) # includes newline pp.stderr.seek(0) for line in pp.stderr.readlines(): + line = str(line, sys.stdout.encoding) if match(line): add(line) if warnings: if config["warnings"]: - with open(config["warnings"], "wb") as f: + with open(config["warnings"], "w") as f: print("".join(warnings), file=f) print("ERROR: %d deprecation warnings found" % len(warnings)) sys.exit(1) diff --git a/newsfragments/3327.minor b/newsfragments/3327.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3873.incompat b/newsfragments/3873.incompat new file mode 100644 index 000000000..da8a5fb0e --- /dev/null +++ b/newsfragments/3873.incompat @@ -0,0 +1 @@ +Python 3.7 or later is now required; Python 2 is no longer supported. \ No newline at end of file diff --git a/pyinstaller.spec b/pyinstaller.spec index 875629c13..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -11,7 +11,10 @@ import struct import sys -if not hasattr(sys, 'real_prefix'): +try: + import allmydata + del allmydata +except ImportError: sys.exit("Please run inside a virtualenv with Tahoe-LAFS installed.") diff --git a/setup.py b/setup.py index c38c0bfec..8bb2b57aa 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,7 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 21.7.0 ; python_version > '3.0'", + "foolscap >= 21.7.0", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] @@ -106,16 +105,10 @@ install_requires = [ # for 'tahoe invite' and 'tahoe join' "magic-wormhole >= 0.10.2", - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - "eliot ~= 1.7 ; python_version < '3.0'", - # On Python 3, we want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0 ; python_version > '3.0'", + # We want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0", - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - "pyrsistent < 0.17.0 ; python_version < '3.0'", - "pyrsistent ; python_version > '3.0'", + "pyrsistent", # A great way to define types of values. "attrs >= 18.2.0", @@ -135,14 +128,8 @@ install_requires = [ # Linux distribution detection: "distro >= 1.4.0", - # Backported configparser for Python 2: - "configparser ; python_version < '3.0'", - - # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. Python - # 2 doesn't actually need this, since HTTP storage protocol isn't supported - # there, so we just pick whatever version so that code imports. - "collections-extended >= 2.0.2 ; python_version > '3.0'", - "collections-extended ; python_version < '3.0'", + # For the RangeMap datastructure. Need 2.0.2 at least for bugfixes. + "collections-extended >= 2.0.2", # HTTP server and client "klein", @@ -201,8 +188,7 @@ trove_classifiers=[ "Natural Language :: English", "Programming Language :: C", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", "Topic :: Utilities", "Topic :: System :: Systems Administration", "Topic :: System :: Filesystems", @@ -229,7 +215,7 @@ def run_command(args, cwd=None): use_shell = sys.platform == "win32" try: p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+ print("Warning: unable to run %r." % (" ".join(args),)) print(e) return None @@ -380,8 +366,8 @@ setup(name="tahoe-lafs", # also set in __init__.py package_dir = {'':'src'}, packages=find_packages('src') + ['allmydata.test.plugins'], classifiers=trove_classifiers, - # We support Python 2.7, and Python 3.7 or later. - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*", + # We support Python 3.7 or later. 3.10 is not supported quite yet. + python_requires=">=3.7, <3.10", install_requires=install_requires, extras_require={ # Duplicate the Twisted pywin32 dependency here. See @@ -400,10 +386,6 @@ setup(name="tahoe-lafs", # also set in __init__.py "tox", "pytest", "pytest-twisted", - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - "decorator < 5", "hypothesis >= 3.6.1", "towncrier", "testtools", diff --git a/tox.ini b/tox.ini index 34d555aa7..9a28b7b30 100644 --- a/tox.ini +++ b/tox.ini @@ -7,8 +7,7 @@ # the tox-gh-actions package. [gh-actions] python = - 2.7: py27-coverage,codechecks - 3.7: py37-coverage,typechecks,codechecks3 + 3.7: py37-coverage,typechecks,codechecks 3.8: py38-coverage 3.9: py39-coverage pypy-3.7: pypy3 @@ -17,7 +16,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,codechecks3,py{27,37,38,39}-{coverage},pypy27,pypy3,integration,integration3 +envlist = typechecks,codechecks,py{37,38,39}-{coverage},pypy27,pypy3,integration minversion = 2.4 [testenv] @@ -35,12 +34,10 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - # - # For now these are versions that support Python 2. - pip==20.3.4 - setuptools==44.1.1 - wheel==0.36.2 - subunitreporter==19.3.2 + pip==22.0.3 + setuptools==60.9.1 + wheel==0.37.1 + subunitreporter==22.2.0 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is # guaranteed to cause things to break eventually as old certificates @@ -89,40 +86,20 @@ commands = coverage: coverage report [testenv:integration] -setenv = - COVERAGE_PROCESS_START=.coveragerc -commands = - # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --timeout=1800 --coverage -v {posargs:integration} - coverage combine - coverage report - - -[testenv:integration3] basepython = python3 +platform = mylinux: linux + mymacos: darwin + mywindows: win32 setenv = COVERAGE_PROCESS_START=.coveragerc commands = - python --version # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report -# Once 2.7 is dropped, this can be removed. It just does flake8 with Python 2 -# since that can give different results than flake8 on Python 3. [testenv:codechecks] -basepython = python2.7 -setenv = - # If no positional arguments are given, try to run the checks on the - # entire codebase, including various pieces of supporting code. - DEFAULT_FILES=src integration static misc setup.py -commands = - flake8 {posargs:{env:DEFAULT_FILES}} - - -[testenv:codechecks3] basepython = python3 deps = # Newer versions of PyLint have buggy configuration @@ -224,16 +201,11 @@ commands = sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] -# We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) -# is broken when this feature is enabled. -install_command = python -m pip install --no-use-pep517 {opts} {packages} extras = deps = {[testenv]deps} packaging - # PyInstaller 4.0 drops Python 2 support. When we finish porting to - # Python 3 we can reconsider this constraint. - pyinstaller < 4.0 + pyinstaller pefile ; platform_system == "Windows" # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build