From f6608255f92ffdd38bda703bc2a7e173ce7d5841 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Sat, 20 Feb 2021 12:14:14 -0500 Subject: [PATCH 01/10] Port control.py to Python 3 --- newsfragments/3605.minor | 0 src/allmydata/control.py | 28 +++++++++++++++++++--------- src/allmydata/util/_python3.py | 1 + 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 newsfragments/3605.minor diff --git a/newsfragments/3605.minor b/newsfragments/3605.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/control.py b/src/allmydata/control.py index 25ce3cea7..7efa174ab 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -1,3 +1,13 @@ +"""Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, time, tempfile from zope.interface import implementer @@ -13,17 +23,17 @@ from twisted.python import log def get_memory_usage(): # this is obviously linux-specific - stat_names = ("VmPeak", - "VmSize", - #"VmHWM", - "VmData") + stat_names = (b"VmPeak", + b"VmSize", + #b"VmHWM", + b"VmData") stats = {} try: - with open("/proc/self/status", "r") as f: + with open("/proc/self/status", "rb") as f: for line in f: - name, right = line.split(":",2) + name, right = line.split(b":",2) if name in stat_names: - assert right.endswith(" kB\n") + assert right.endswith(b" kB\n") right = right[:-4] stats[name] = int(right) * 1024 except: @@ -34,8 +44,8 @@ def get_memory_usage(): def log_memory_usage(where=""): stats = get_memory_usage() - log.msg("VmSize: %9d VmPeak: %9d %s" % (stats["VmSize"], - stats["VmPeak"], + log.msg("VmSize: %9d VmPeak: %9d %s" % (stats[b"VmSize"], + stats[b"VmPeak"], where)) @implementer(IConsumer) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 51496537a..60ccb91aa 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -29,6 +29,7 @@ PORTED_MODULES = [ "allmydata._monkeypatch", "allmydata.blacklist", "allmydata.codec", + "allmydata.control", "allmydata.crypto", "allmydata.crypto.aes", "allmydata.crypto.ed25519", From 8faea9162fb2df5110bfd461fd3866dcebdceffc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 08:58:14 -0500 Subject: [PATCH 02/10] news fragment --- newsfragments/3623.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3623.minor diff --git a/newsfragments/3623.minor b/newsfragments/3623.minor new file mode 100644 index 000000000..e69de29bb From 3c047f0d17067070143a3b88910643c2cfd81418 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 08:58:53 -0500 Subject: [PATCH 03/10] Set an upper bound on the Twisted we'll use --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5dc68d367..11a55e0d7 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,10 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", + # + # * Twisted 21.2.0 introduces some type hints which we are not yet + # compatible with. + "Twisted[tls,conch] >= 19.10.0, <21.2.0", "PyYAML >= 3.11", From 9f6965ce677458c55dc852ea36344bb66ed34f99 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 10:46:00 -0500 Subject: [PATCH 04/10] [drop] force images to rebuild --- .circleci/config.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b00bcdcec..417d6dca4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,19 +89,19 @@ workflows: - "typechecks": <<: *DOCKERHUB_CONTEXT - images: - # Build the Docker images used by the ci jobs. This makes the ci jobs - # faster and takes various spurious failures out of the critical path. - triggers: - # Build once a day - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - "master" + # images: + # # Build the Docker images used by the ci jobs. This makes the ci jobs + # # faster and takes various spurious failures out of the critical path. + # triggers: + # # Build once a day + # - schedule: + # cron: "0 0 * * *" + # filters: + # branches: + # only: + # - "master" - jobs: + # jobs: - "build-image-debian-10": <<: *DOCKERHUB_CONTEXT - "build-image-debian-9": From 386784110f2842961bf5bffc2ddefdf7987fe90e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 12:46:44 -0500 Subject: [PATCH 05/10] Maybe this way works --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 2ff83cbd4..509244e16 100644 --- a/tox.ini +++ b/tox.ini @@ -120,6 +120,7 @@ deps = mypy git+https://github.com/Shoobx/mypy-zope git+https://github.com/warner/foolscap + twisted<21.2.0 commands = mypy src From e0339b448dfee0f0f56e7d5c55ad19ee985dbaed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 12:49:20 -0500 Subject: [PATCH 06/10] Pin it in the tox configuration Because why should setup.py matter? --- .circleci/config.yml | 24 ++++++++++++------------ setup.py | 5 +---- tox.ini | 3 +++ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 417d6dca4..b00bcdcec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,19 +89,19 @@ workflows: - "typechecks": <<: *DOCKERHUB_CONTEXT - # images: - # # Build the Docker images used by the ci jobs. This makes the ci jobs - # # faster and takes various spurious failures out of the critical path. - # triggers: - # # Build once a day - # - schedule: - # cron: "0 0 * * *" - # filters: - # branches: - # only: - # - "master" + images: + # Build the Docker images used by the ci jobs. This makes the ci jobs + # faster and takes various spurious failures out of the critical path. + triggers: + # Build once a day + - schedule: + cron: "0 0 * * *" + filters: + branches: + only: + - "master" - # jobs: + jobs: - "build-image-debian-10": <<: *DOCKERHUB_CONTEXT - "build-image-debian-9": diff --git a/setup.py b/setup.py index 11a55e0d7..5dc68d367 100644 --- a/setup.py +++ b/setup.py @@ -96,10 +96,7 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - # - # * Twisted 21.2.0 introduces some type hints which we are not yet - # compatible with. - "Twisted[tls,conch] >= 19.10.0, <21.2.0", + "Twisted[tls,conch] >= 19.10.0", "PyYAML >= 3.11", diff --git a/tox.ini b/tox.ini index 509244e16..9b954cb30 100644 --- a/tox.ini +++ b/tox.ini @@ -120,6 +120,9 @@ deps = mypy git+https://github.com/Shoobx/mypy-zope git+https://github.com/warner/foolscap + # Twisted 21.2.0 introduces some type hints which we are not yet + # compatible with. + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3622 twisted<21.2.0 commands = mypy src From 91ccf33829836e22de0dd0601b4a5655f01147f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 13:29:31 -0500 Subject: [PATCH 07/10] news fragment --- newsfragments/3608.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3608.minor diff --git a/newsfragments/3608.minor b/newsfragments/3608.minor new file mode 100644 index 000000000..e69de29bb From 7c9e757f70546645d02cf3dcefda7b4041ccf9b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 13:29:36 -0500 Subject: [PATCH 08/10] go back to a released txi2p for python 2 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5dc68d367..34f16e45f 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,8 @@ i2p_requires = [ # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. # URL lookups are in PEP-508 (via https://stackoverflow.com/a/54794506). # Also see the comment in tor_requires. - "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p", + "txi2p; python_version < '3.0'", + "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p; python_version > '3.0'", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From 7b82a1230f82d3ecbbe227ade3610615518a3e1a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 13:47:39 -0500 Subject: [PATCH 09/10] Drop txi2p for Python 3 entirely PEP-508 "URL lookups" are unsupported in packages released on PyPI --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 34f16e45f..ed48bf789 100644 --- a/setup.py +++ b/setup.py @@ -152,10 +152,7 @@ tor_requires = [ i2p_requires = [ # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. - # URL lookups are in PEP-508 (via https://stackoverflow.com/a/54794506). - # Also see the comment in tor_requires. "txi2p; python_version < '3.0'", - "txi2p @ git+https://github.com/str4d/txi2p@0611b9a86172cb70d2f5e415a88eee9f230590b3#egg=txi2p; python_version > '3.0'", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From dae40c7f867de3da0397f3a82827126cf6ac3c49 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 1 Mar 2021 14:11:17 -0500 Subject: [PATCH 10/10] pointless twiddle CircleCI keeps failing to fetch the revision for some jobs Try giving it a new revision???? --- newsfragments/3623.minor | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3623.minor b/newsfragments/3623.minor index e69de29bb..8b1378917 100644 --- a/newsfragments/3623.minor +++ b/newsfragments/3623.minor @@ -0,0 +1 @@ +