Merge pull request #1053 from tahoe-lafs/3683.newer-python3-versions

Test newer versions of Python 3 on Linux and macOS

Fixes ticket:3683

.github/workflows/ci.yml

@@ -19,11 +19,18 @@ jobs:
       matrix:
         os:
           - macos-latest
-          - windows-latest
           - ubuntu-latest
         python-version:
           - 2.7
           - 3.6
+          - 3.7
+          - 3.8
+          - 3.9
+        include:
+          # For now we're only doing Windows on 2.7, will be fixed in
+          # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701
+          - os: windows-latest
+            python-version: 2.7
 
     steps:
 
@@ -36,17 +43,15 @@ jobs:
 
       - name: Set up Python ${{ matrix.python-version }}
         if: ${{ matrix.os != 'windows-latest' }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
       # See note below about need for using 32-bit Python 2.7 on
-      # Windows.  The extra handling here for Python 3.6 on Windows is
-      # because I could not figure out the right GitHub Actions
-      # expression to do this in a better way.
+      # Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x64]
-        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }}
-        uses: actions/setup-python@v1
+        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version != '2.7' ) }}
+        uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
           architecture: 'x64'
@@ -85,14 +90,14 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade codecov tox setuptools
+          pip install --upgrade codecov tox tox-gh-actions setuptools
           pip list
 
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e py27-coverage"
-        run: tox -e py27-coverage
+      - name: Run tox for corresponding Python version
+        run: python -m tox
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1

src/allmydata/scripts/tahoe_backup.py

@@ -22,7 +22,6 @@ def get_local_metadata(path):
     metadata = {}
 
     # posix stat(2) metadata, depends on the platform
-    os.stat_float_times(True)
     s = os.stat(path)
     metadata["ctime"] = s.st_ctime
     metadata["mtime"] = s.st_mtime

src/allmydata/stats.py

@@ -9,7 +9,9 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
+    from time import clock as process_time
+else:
+    from time import process_time
 import time
 
 from twisted.application import service
@@ -27,7 +29,7 @@ class CPUUsageMonitor(service.MultiService):
 
     def __init__(self):
         service.MultiService.__init__(self)
-        # we don't use time.clock() here, because the constructor is run by
+        # we don't use process_time() here, because the constructor is run by
         # the twistd parent process (as it loads the .tac file), whereas the
         # rest of the program will be run by the child process, after twistd
         # forks. Instead, set self.initial_cpu as soon as the reactor starts
@@ -39,11 +41,11 @@ class CPUUsageMonitor(service.MultiService):
         TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self)
 
     def _set_initial_cpu(self):
-        self.initial_cpu = time.clock()
+        self.initial_cpu = process_time()
 
     def check(self):
         now_wall = time.time()
-        now_cpu = time.clock()
+        now_cpu = process_time()
         self.samples.append( (now_wall, now_cpu) )
         while len(self.samples) > self.HISTORY_LENGTH+1:
             self.samples.pop(0)
@@ -68,7 +70,7 @@ class CPUUsageMonitor(service.MultiService):
         avg = self._average_N_minutes(15)
         if avg is not None:
             s["cpu_monitor.15min_avg"] = avg
-        now_cpu = time.clock()
+        now_cpu = process_time()
         s["cpu_monitor.total"] = now_cpu - self.initial_cpu
         return s
 

src/allmydata/test/common.py

@@ -405,7 +405,7 @@ class SameProcessStreamEndpointAssigner(object):
         :return: A two-tuple of (location hint, port endpoint description) as
             strings.
         """
-        if IReactorSocket.providedBy(reactor):
+        if sys.platform != "win32" and IReactorSocket.providedBy(reactor):
             # On this platform, we can reliable pre-allocate a listening port.
             # Once it is bound we know it will not fail later with EADDRINUSE.
             s = socket(AF_INET, SOCK_STREAM)

src/allmydata/test/test_util.py

@@ -527,7 +527,7 @@ class JSONBytes(unittest.TestCase):
         x = {u"def\N{SNOWMAN}\uFF00": 123}
         encoded = jsonbytes.dumps_bytes(x)
         self.assertIsInstance(encoded, bytes)
-        self.assertEqual(json.loads(encoded, encoding="utf-8"), x)
+        self.assertEqual(json.loads(encoded), x)
 
     def test_any_bytes_unsupported_by_default(self):
         """By default non-UTF-8 bytes raise error."""
@@ -554,9 +554,8 @@ class JSONBytes(unittest.TestCase):
             expected,
         )
         self.assertEqual(
-            json.loads(jsonbytes.dumps(o, any_bytes=True),
-                       encoding="utf-8"),
-            expected,
+            json.loads(jsonbytes.dumps(o, any_bytes=True)),
+            expected
         )
 
 

src/allmydata/windows/fixups.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+from past.builtins import unicode
 
 # This code isn't loadable or sensible except on Windows.  Importers all know
 # this and are careful.  Normally I would just let an import error from ctypes

src/allmydata/windows/registry.py

@@ -1,5 +1,5 @@
 import sys
-import _winreg
+import winreg
 
 _AMD_KEY = r"Software\Allmydata"
 _BDIR_KEY = 'Base Dir Path'
@@ -22,19 +22,19 @@ def get_registry_setting(key, name, _topkey=None):
     @param name: The name of the setting we are querying.
     @type name: String
     """
-    topkeys = [_winreg.HKEY_CURRENT_USER, _winreg.HKEY_LOCAL_MACHINE]
+    topkeys = [winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE]
 
     if _topkey:
         topkeys.insert(0, _topkey)
 
     for topkey in topkeys:
         try:
-            regkey = _winreg.OpenKey(topkey, key)
+            regkey = winreg.OpenKey(topkey, key)
 
-            sublen, vallen, timestamp = _winreg.QueryInfoKey(regkey)
+            sublen, vallen, timestamp = winreg.QueryInfoKey(regkey)
             for validx in xrange(vallen):
-                keyname, value, keytype = _winreg.EnumValue(regkey, validx)
-                if keyname == name and keytype == _winreg.REG_SZ:
+                keyname, value, keytype = winreg.EnumValue(regkey, validx)
+                if keyname == name and keytype == winreg.REG_SZ:
                     return value
 
         except WindowsError:
@@ -42,27 +42,27 @@ def get_registry_setting(key, name, _topkey=None):
     # We didn't find the key:
     raise KeyError(key, name, "registry setting not found")
 
-def set_registry_setting(key, name, data, reg_type=_winreg.REG_SZ,
-                         _topkey=_winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True):
+def set_registry_setting(key, name, data, reg_type=winreg.REG_SZ,
+                         _topkey=winreg.HKEY_LOCAL_MACHINE, create_key_if_missing=True):
     """
     Sets a registry setting.
 
     defaults to string values (REG_SZ) - overridable with reg_type.
     """
     try:
-        regkey = _winreg.OpenKey(_topkey, key, 0, _winreg.KEY_SET_VALUE)
+        regkey = winreg.OpenKey(_topkey, key, 0, winreg.KEY_SET_VALUE)
     except WindowsError:
         if create_key_if_missing:
-            regkey = _winreg.CreateKey(_topkey, key)
+            regkey = winreg.CreateKey(_topkey, key)
         else:
             raise KeyError(key, "registry key not found")
 
     try:
-        _winreg.DeleteValue(regkey, name)
+        winreg.DeleteValue(regkey, name)
     except:
         pass
 
-    _winreg.SetValueEx(regkey, name, 0, reg_type, data)
+    winreg.SetValueEx(regkey, name, 0, reg_type, data)
 
 def get_registry_value(keyname):
     """

tox.ini

@@ -3,11 +3,21 @@
 # test suite on all supported python versions. To use it, "pip install tox"
 # and then run "tox" from this directory.
 
+# Map Python versions in GitHub Actions to tox environments to run.
+[gh-actions]
+python =
+    2.7: py27-coverage,codechecks
+    3.6: py36-coverage
+    3.7: py37-coverage
+    3.8: py38-coverage
+    3.9: py39-coverage,typechecks
+    pypy-3.7: pypy3
+
 [pytest]
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,py27,py36,pypy27
+envlist = typechecks,codechecks,py{27,36,37,38,39}-{coverage},pypy27,pypy3
 minversion = 2.4
 
 [testenv]
@@ -25,9 +35,11 @@ deps =
      # happening at the time.  The versions selected here are just the current
      # versions at the time.  Bumping them to keep up with future releases is
      # fine as long as those releases are known to actually work.
-     pip==19.1.1
-     setuptools==41.0.1
-     wheel==0.33.4
+     #
+     # For now these are versions that support Python 2.
+     pip==20.3.4
+     setuptools==44.1.1
+     wheel==0.36.2
      subunitreporter==19.3.2
      # As an exception, we don't pin certifi because it contains CA
      # certificates which necessarily change over time.  Pinning this is