From 8f997870c8db9a081bf6f9f06f18a53ce7f14345 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 09:47:28 -0400 Subject: [PATCH 01/15] Test newer versions of Python 3, as well as PyPy 3. --- .github/workflows/ci.yml | 4 ++++ newsfragments/3683.minor | 0 2 files changed, 4 insertions(+) create mode 100644 newsfragments/3683.minor diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f70432267..47e88a9a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,10 @@ jobs: python-version: - 2.7 - 3.6 + - 3.7 + - 3.8 + - 3.9 + - pypy-3.7 steps: diff --git a/newsfragments/3683.minor b/newsfragments/3683.minor new file mode 100644 index 000000000..e69de29bb From ea2db971a6f98f3ff6c3d269c750b2aa13e3e3c2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:06:35 -0400 Subject: [PATCH 02/15] Try to make GitHub Actions automatically use appropriate Tox envs by using `tox-gh-actions` package. --- .github/workflows/ci.yml | 16 +++++++--------- tox.ini | 12 +++++++++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47e88a9a0..b60b51b4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,17 +40,15 @@ jobs: - name: Set up Python ${{ matrix.python-version }} if: ${{ matrix.os != 'windows-latest' }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} # See note below about need for using 32-bit Python 2.7 on - # Windows. The extra handling here for Python 3.6 on Windows is - # because I could not figure out the right GitHub Actions - # expression to do this in a better way. + # Windows. - name: Set up Python ${{ matrix.python-version }} [Windows x64] - if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }} - uses: actions/setup-python@v1 + if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version != '2.7' ) }} + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} architecture: 'x64' @@ -89,14 +87,14 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools + pip install --upgrade codecov tox tox-gh-actions setuptools pip list - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py - - name: Run "tox -e py27-coverage" - run: tox -e py27-coverage + - name: Run tox for corresponding Python version + run: python -m tox - name: Upload eliot.log in case of failure uses: actions/upload-artifact@v1 diff --git a/tox.ini b/tox.ini index a58d9d447..73dbb8e78 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,21 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# Map Python versions in GitHub Actions to tox environments to run. +[gh-actions] +python = + 2.7: py27-coverage,codechecks + 3.6: py36-coverage + 3.7: py37-coverage + 3.8: py38-coverage + 3.9: py39-coverage,typechecks + pypy-3.7: pypy3 + [pytest] twisted = 1 [tox] -envlist = typechecks,codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,py37,py38,py39,pypy27,pypy3 minversion = 2.4 [testenv] From ebedc660bd244e8ed0d5c7a06200a0c0d7486ef4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:12:42 -0400 Subject: [PATCH 03/15] Note coverage explicitly in envlist. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 73dbb8e78..2274b42e7 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ python = twisted = 1 [tox] -envlist = typechecks,codechecks,py27,py36,py37,py38,py39,pypy27,pypy3 +envlist = typechecks,codechecks,py{27,36,37,38,39}-{coverage},pypy27,pypy3 minversion = 2.4 [testenv] From 293f372ea44118e2e9217d9ef161ac739dfe2748 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:17:13 -0400 Subject: [PATCH 04/15] Version of Twisted that doesn't break on Python 3 with Windows. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df917eb40..32b7f25d2 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,8 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", + # * Twisted 21.2.0 is missing binary wheels for some reason. + "Twisted[tls,conch] >= 19.10.0, != 21.2.0", "PyYAML >= 3.11", From 30c03d085e352baf7419ff6644e7b6411ab86e24 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:34:10 -0400 Subject: [PATCH 05/15] Newer versions of setuptools and pip, to work better with newer Pythons. --- tox.ini | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 2274b42e7..4b52fd316 100644 --- a/tox.ini +++ b/tox.ini @@ -35,9 +35,11 @@ deps = # happening at the time. The versions selected here are just the current # versions at the time. Bumping them to keep up with future releases is # fine as long as those releases are known to actually work. - pip==19.1.1 - setuptools==41.0.1 - wheel==0.33.4 + # + # For now these are versions that support Python 2. + pip==20.3.4 + setuptools==44.1.1 + wheel==0.36.2 subunitreporter==19.3.2 # As an exception, we don't pin certifi because it contains CA # certificates which necessarily change over time. Pinning this is From 65159c99616daa1c9d2b54299d9c01f5e1ebfee7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 10:37:49 -0400 Subject: [PATCH 06/15] Use modern (and actually correct on Windows!) API for getting CPU time. --- src/allmydata/stats.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 7137ba28e..13ed8817c 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -9,7 +9,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - + from time import clock as process_time +else: + from time import process_time import time from twisted.application import service @@ -27,7 +29,7 @@ class CPUUsageMonitor(service.MultiService): def __init__(self): service.MultiService.__init__(self) - # we don't use time.clock() here, because the constructor is run by + # we don't use process_time() here, because the constructor is run by # the twistd parent process (as it loads the .tac file), whereas the # rest of the program will be run by the child process, after twistd # forks. Instead, set self.initial_cpu as soon as the reactor starts @@ -39,11 +41,11 @@ class CPUUsageMonitor(service.MultiService): TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self) def _set_initial_cpu(self): - self.initial_cpu = time.clock() + self.initial_cpu = process_time() def check(self): now_wall = time.time() - now_cpu = time.clock() + now_cpu = process_time() self.samples.append( (now_wall, now_cpu) ) while len(self.samples) > self.HISTORY_LENGTH+1: self.samples.pop(0) @@ -68,7 +70,7 @@ class CPUUsageMonitor(service.MultiService): avg = self._average_N_minutes(15) if avg is not None: s["cpu_monitor.15min_avg"] = avg - now_cpu = time.clock() + now_cpu = process_time() s["cpu_monitor.total"] = now_cpu - self.initial_cpu return s From 1ed24a16169b77c844affed59101cc7b896f4a7d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:22:17 -0400 Subject: [PATCH 07/15] This API is irrelevant in Python 2.5 or later, and not present in newer Python 3. --- src/allmydata/scripts/tahoe_backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 2f3bbeed1..4847f2e8c 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -22,7 +22,6 @@ def get_local_metadata(path): metadata = {} # posix stat(2) metadata, depends on the platform - os.stat_float_times(True) s = os.stat(path) metadata["ctime"] = s.st_ctime metadata["mtime"] = s.st_mtime From fa4be104a63fbeb011b3f25858138b8e5004ad53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:31:19 -0400 Subject: [PATCH 08/15] New name for module. --- src/allmydata/windows/registry.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/windows/registry.py b/src/allmydata/windows/registry.py index c1f695d2b..2c7f03156 100644 --- a/src/allmydata/windows/registry.py +++ b/src/allmydata/windows/registry.py @@ -1,5 +1,5 @@ import sys -import _winreg +import winreg _AMD_KEY = r"Software\Allmydata" _BDIR_KEY = 'Base Dir Path' @@ -22,19 +22,19 @@ def get_registry_setting(key, name, _topkey=None): @param name: The name of the setting we are querying. @type name: String """ - topkeys = [_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE] + topkeys = [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE] if _topkey: topkeys.insert(0, _topkey) for topkey in topkeys: try: - regkey = _winreg.OpenKey(topkey, key) + regkey = winreg.OpenKey(topkey, key) - sublen, vallen, timestamp = _winreg.QueryInfoKey(regkey) + sublen, vallen, timestamp = winreg.QueryInfoKey(regkey) for validx in xrange(vallen): - keyname, value, keytype = _winreg.EnumValue(regkey, validx) - if keyname == name and keytype == _winreg.REG_SZ: + keyname, value, keytype = winreg.EnumValue(regkey, validx) + if keyname == name and keytype == winreg.REG_SZ: return value except WindowsError: @@ -42,27 +42,27 @@ def get_registry_setting(key, name, _topkey=None): # We didn't find the key: raise KeyError(key, name, "registry setting not found") -def set_registry_setting(key, name, data, reg_type=_winreg.REG_SZ, - _topkey=_winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): +def set_registry_setting(key, name, data, reg_type=winreg.REG_SZ, + _topkey=winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True): """ Sets a registry setting. defaults to string values (REG_SZ) - overridable with reg_type. """ try: - regkey = _winreg.OpenKey(_topkey, key, 0, _winreg.KEY_SET_VALUE) + regkey = winreg.OpenKey(_topkey, key, 0, winreg.KEY_SET_VALUE) except WindowsError: if create_key_if_missing: - regkey = _winreg.CreateKey(_topkey, key) + regkey = winreg.CreateKey(_topkey, key) else: raise KeyError(key, "registry key not found") try: - _winreg.DeleteValue(regkey, name) + winreg.DeleteValue(regkey, name) except: pass - _winreg.SetValueEx(regkey, name, 0, reg_type, data) + winreg.SetValueEx(regkey, name, 0, reg_type, data) def get_registry_value(keyname): """ From fa0ec99052cd920da59272f572a03f0c290692d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:43:59 -0400 Subject: [PATCH 09/15] Maybe newer Twisted is OK, IOCP was supposedly moved out. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 32b7f25d2..df917eb40 100644 --- a/setup.py +++ b/setup.py @@ -97,8 +97,7 @@ install_requires = [ # an sftp extra in Tahoe-LAFS, there is no point in having one. # * Twisted 19.10 introduces Site.getContentFile which we use to get # temporary upload files placed into a per-node temporary directory. - # * Twisted 21.2.0 is missing binary wheels for some reason. - "Twisted[tls,conch] >= 19.10.0, != 21.2.0", + "Twisted[tls,conch] >= 19.10.0", "PyYAML >= 3.11", From 8af84b7a0cc3c84848e181b933c1e3c4f4f141f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:57:13 -0400 Subject: [PATCH 10/15] Fix tests on Python 3.9 (apparently encoding is no-op on Python 3, and Python always assumes utf-8 by default anyway for JSON). --- src/allmydata/test/test_util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 4c2e98683..a03845ed6 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -527,7 +527,7 @@ class JSONBytes(unittest.TestCase): x = {u"def\N{SNOWMAN}\uFF00": 123} encoded = jsonbytes.dumps_bytes(x) self.assertIsInstance(encoded, bytes) - self.assertEqual(json.loads(encoded, encoding="utf-8"), x) + self.assertEqual(json.loads(encoded), x) def test_any_bytes_unsupported_by_default(self): """By default non-UTF-8 bytes raise error.""" @@ -554,9 +554,8 @@ class JSONBytes(unittest.TestCase): expected, ) self.assertEqual( - json.loads(jsonbytes.dumps(o, any_bytes=True), - encoding="utf-8"), - expected, + json.loads(jsonbytes.dumps(o, any_bytes=True)), + expected ) From f645715b152e2e0c30b20431e0b404a05c128ad2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 11:59:06 -0400 Subject: [PATCH 11/15] Put off pypy3 to a later ticket. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b60b51b4d..4644b84c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: - 3.7 - 3.8 - 3.9 - - pypy-3.7 steps: From da5c38b7ca318430198eeafe8742b20130e87580 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 12:39:42 -0400 Subject: [PATCH 12/15] Fix a NameError. --- src/allmydata/windows/fixups.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index 3b5437a7b..732a792f5 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -1,4 +1,5 @@ from __future__ import print_function +from past.builtins import unicode # This code isn't loadable or sensible except on Windows. Importers all know # this and are careful. Normally I would just let an import error from ctypes From 4ff8a2a09ca4776e1c06f3ef5b8a132c08775e75 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 12:59:06 -0400 Subject: [PATCH 13/15] Pretty sure reactor implements IReactorSocket on Windows on Python 3, but that's still not enough for this functionality to work (and it might be a Twisted bug). --- src/allmydata/test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d874d07ae..0f2dc7c62 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -405,7 +405,7 @@ class SameProcessStreamEndpointAssigner(object): :return: A two-tuple of (location hint, port endpoint description) as strings. """ - if IReactorSocket.providedBy(reactor): + if sys.platform != "win32" and IReactorSocket.providedBy(reactor): # On this platform, we can reliable pre-allocate a listening port. # Once it is bound we know it will not fail later with EADDRINUSE. s = socket(AF_INET, SOCK_STREAM) From 7c598d6f028ff0027db720a41179003eb782c666 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 13:00:29 -0400 Subject: [PATCH 14/15] News file. --- newsfragments/3676.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3676.minor diff --git a/newsfragments/3676.minor b/newsfragments/3676.minor new file mode 100644 index 000000000..e69de29bb From 8902868b907b1f9a64886081fcab6a8cee7719b0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 May 2021 13:15:29 -0400 Subject: [PATCH 15/15] Skip Windows Python 3 for now. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4644b84c7..1f3d57310 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,6 @@ jobs: matrix: os: - macos-latest - - windows-latest - ubuntu-latest python-version: - 2.7 @@ -27,6 +26,11 @@ jobs: - 3.7 - 3.8 - 3.9 + include: + # For now we're only doing Windows on 2.7, will be fixed in + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701 + - os: windows-latest + python-version: 2.7 steps: