Merge pull request #1067 from tahoe-lafs/3700.byteswarning-windows

Run (and pass) Python 3 unit tests on Windows

Fixes ticket:3700
Fixes ticket:3701

.github/workflows/ci.yml

@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
+          - windows-latest
           - macos-latest
           - ubuntu-latest
         python-version:
@@ -26,11 +27,6 @@ jobs:
           - 3.7
           - 3.8
           - 3.9
-        include:
-          # For now we're only doing Windows on 2.7, will be fixed in
-          # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3701
-          - os: windows-latest
-            python-version: 2.7
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0

src/allmydata/test/cli/test_backup.py

@@ -6,17 +6,19 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2
+from future.utils import PY2, PY3
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
     import __builtin__ as builtins
 else:
     import builtins
+from six import ensure_str
 
 import os.path
 from six.moves import cStringIO as StringIO
 from datetime import timedelta
 import re
+import locale
 
 from twisted.trial import unittest
 from twisted.python.monkey import MonkeyPatcher
@@ -374,7 +376,9 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
     def test_exclude_options_unicode(self):
         nice_doc = u"nice_d\u00F8c.lyx"
         try:
-            doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding())
+            doc_pattern_arg = u"*d\u00F8c*"
+            if PY2:
+                doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding())
         except UnicodeEncodeError:
             raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
 
@@ -396,7 +400,11 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase):
         self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'),
                              (nice_doc, u'lib.a'))
         # read exclude patterns from file
-        exclusion_string = doc_pattern_arg + b"\nlib.?"
+        exclusion_string = doc_pattern_arg + ensure_str("\nlib.?")
+        if PY3:
+            # On Python 2 this gives some garbage encoding. Also on Python 2 we
+            # expect exclusion string to be bytes.
+            exclusion_string = exclusion_string.encode(locale.getpreferredencoding(False))
         excl_filepath = os.path.join(basedir, 'exclusion')
         fileutil.write(excl_filepath, exclusion_string)
         backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to'])

src/allmydata/test/test_runner.py

@@ -17,6 +17,7 @@ from six import ensure_text
 
 import os.path, re, sys
 from os import linesep
+import locale
 
 from eliot import (
     log_call,
@@ -92,8 +93,12 @@ def run_bintahoe(extra_argv, python_options=None):
     argv.extend(extra_argv)
     argv = list(unicode_to_argv(arg) for arg in argv)
     p = Popen(argv, stdout=PIPE, stderr=PIPE)
-    out = p.stdout.read().decode("utf-8")
-    err = p.stderr.read().decode("utf-8")
+    if PY2:
+        encoding = "utf-8"
+    else:
+        encoding = locale.getpreferredencoding(False)
+    out = p.stdout.read().decode(encoding)
+    err = p.stderr.read().decode(encoding)
     returncode = p.wait()
     return (out, err, returncode)
 
@@ -103,7 +108,7 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase):
         """
         The runner script receives unmangled non-ASCII values in argv.
         """
-        tricky = u"\u2621"
+        tricky = u"\u00F6"
         out, err, returncode = run_bintahoe([tricky])
         self.assertEqual(returncode, 1)
         self.assertIn(u"Unknown command: " + tricky, out)

src/allmydata/test/test_windows.py

@@ -79,6 +79,7 @@ slow_settings = settings(
 )
 
 @skipUnless(platform.isWindows(), "get_argv is Windows-only")
+@skipUnless(PY2, "Not used on Python 3.")
 class GetArgvTests(SyncTestCase):
     """
     Tests for ``get_argv``.
@@ -172,6 +173,7 @@ class GetArgvTests(SyncTestCase):
 
 
 @skipUnless(platform.isWindows(), "intended for Windows-only codepaths")
+@skipUnless(PY2, "Not used on Python 3.")
 class UnicodeOutputTests(SyncTestCase):
     """
     Tests for writing unicode to stdout and stderr.

src/allmydata/windows/fixups.py

@@ -1,4 +1,6 @@
 from __future__ import print_function
+
+from future.utils import PY3
 from past.builtins import unicode
 
 # This code isn't loadable or sensible except on Windows.  Importers all know
@@ -122,6 +124,10 @@ def initialize():
 
     SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
 
+    if PY3:
+        # The rest of this appears to be Python 2-specific
+        return
+
     original_stderr = sys.stderr
 
     # If any exception occurs in this code, we'll probably try to print it on stderr,

tox.ini

@@ -9,9 +9,9 @@
 python =
     2.7: py27-coverage,codechecks
     3.6: py36-coverage
-    3.7: py37-coverage
+    3.7: py37-coverage,typechecks,codechecks3
     3.8: py38-coverage
-    3.9: py39-coverage,typechecks,codechecks3
+    3.9: py39-coverage
     pypy-3.7: pypy3
 
 [pytest]