Merge pull request #1034 from tahoe-lafs/3667.test-utilities-python-3

Port more of allmydata.test utility modules to Python 3

Fixes ticket:3667

src/allmydata/test/_twisted_9607.py

@@ -1,74 +0,0 @@
-"""
-A copy of the implementation of Twisted's ``getProcessOutputAndValue``
-with the fix for Twisted #9607 (support for stdinBytes) patched in.
-"""
-
-from __future__ import (
-    division,
-    absolute_import,
-    print_function,
-    unicode_literals,
-)
-
-from io import BytesIO
-
-from twisted.internet import protocol, defer
-
-
-class _EverythingGetter(protocol.ProcessProtocol, object):
-
-    def __init__(self, deferred, stdinBytes=None):
-        self.deferred = deferred
-        self.outBuf = BytesIO()
-        self.errBuf = BytesIO()
-        self.outReceived = self.outBuf.write
-        self.errReceived = self.errBuf.write
-        self.stdinBytes = stdinBytes
-
-    def connectionMade(self):
-        if self.stdinBytes is not None:
-            self.transport.writeToChild(0, self.stdinBytes)
-            # The only compelling reason not to _always_ close stdin here is
-            # backwards compatibility.
-            self.transport.closeStdin()
-
-    def processEnded(self, reason):
-        out = self.outBuf.getvalue()
-        err = self.errBuf.getvalue()
-        e = reason.value
-        code = e.exitCode
-        if e.signal:
-            self.deferred.errback((out, err, e.signal))
-        else:
-            self.deferred.callback((out, err, code))
-
-
-
-def _callProtocolWithDeferred(protocol, executable, args, env, path,
-                              reactor=None, protoArgs=()):
-    if reactor is None:
-        from twisted.internet import reactor
-
-    d = defer.Deferred()
-    p = protocol(d, *protoArgs)
-    reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path)
-    return d
-
-
-
-def getProcessOutputAndValue(executable, args=(), env={}, path=None,
-                             reactor=None, stdinBytes=None):
-    """Spawn a process and returns a Deferred that will be called back with
-    its output (from stdout and stderr) and it's exit code as (out, err, code)
-    If a signal is raised, the Deferred will errback with the stdout and
-    stderr up to that point, along with the signal, as (out, err, signalNum)
-    """
-    return _callProtocolWithDeferred(
-        _EverythingGetter,
-        executable,
-        args,
-        env,
-        path,
-        reactor,
-        protoArgs=(stdinBytes,),
-    )

src/allmydata/test/cli/test_status.py

@@ -37,10 +37,26 @@ from allmydata.util import jsonbytes as json
 
 from ..no_network import GridTestMixin
 from ..common_web import do_http
-from ..status import FakeStatus
 from .common import CLITestMixin
 
 
+class FakeStatus(object):
+    def __init__(self):
+        self.status = []
+
+    def setServiceParent(self, p):
+        pass
+
+    def get_status(self):
+        return self.status
+
+    def get_storage_index(self):
+        return None
+
+    def get_size(self):
+        return None
+
+
 class ProgressBar(unittest.TestCase):
 
     def test_ascii0(self):

src/allmydata/test/cli_node_api.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 __all__ = [
     "CLINodeAPI",
@@ -81,7 +92,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object):
         self._fds = fds
 
     def connectionMade(self):
-        for proto in self._fds.values():
+        for proto in list(self._fds.values()):
             proto.makeConnection(self.transport)
 
     def childDataReceived(self, childFD, data):
@@ -94,7 +105,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object):
 
     def processEnded(self, reason):
         notified = set()
-        for proto in self._fds.values():
+        for proto in list(self._fds.values()):
             if proto not in notified:
                 proto.connectionLost(reason)
                 notified.add(proto)

src/allmydata/test/common.py

@@ -1,4 +1,15 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2, native_str
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from past.builtins import chr as byteschr
 
 __all__ = [
     "SyncTestCase",
@@ -15,8 +26,6 @@ __all__ = [
     "PIPE",
 ]
 
-from past.builtins import chr as byteschr, unicode
-
 import sys
 import os, random, struct
 import six
@@ -106,7 +115,7 @@ from .eliotutil import (
 )
 from .common_util import ShouldFailMixin  # noqa: F401
 
-if sys.platform == "win32":
+if sys.platform == "win32" and PY2:
     # Python 2.7 doesn't have good options for launching a process with
     # non-ASCII in its command line.  So use this alternative that does a
     # better job.  However, only use it on Windows because it doesn't work
@@ -253,7 +262,7 @@ class UseNode(object):
     plugin_config = attr.ib()
     storage_plugin = attr.ib()
     basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
-    introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
+    introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str),
                               converter=six.ensure_str)
     node_config = attr.ib(default=attr.Factory(dict))
 
@@ -264,7 +273,7 @@ class UseNode(object):
             return "\n".join(
                 " = ".join((key, value))
                 for (key, value)
-                in config.items()
+                in list(config.items())
             )
 
         if self.plugin_config is None:
@@ -849,17 +858,17 @@ class WebErrorMixin(object):
                         callable=None, *args, **kwargs):
         # returns a Deferred with the response body
         if isinstance(substring, bytes):
-            substring = unicode(substring, "ascii")
-        if isinstance(response_substring, unicode):
+            substring = str(substring, "ascii")
+        if isinstance(response_substring, str):
             response_substring = response_substring.encode("ascii")
-        assert substring is None or isinstance(substring, unicode)
+        assert substring is None or isinstance(substring, str)
         assert response_substring is None or isinstance(response_substring, bytes)
         assert callable
         def _validate(f):
             if code is not None:
                 self.failUnlessEqual(f.value.status, b"%d" % code, which)
             if substring:
-                code_string = unicode(f)
+                code_string = str(f)
                 self.failUnless(substring in code_string,
                                 "%s: substring '%s' not in '%s'"
                                 % (which, substring, code_string))
@@ -882,7 +891,7 @@ class WebErrorMixin(object):
         body = yield response.content()
         self.assertEquals(response.code, code)
         if response_substring is not None:
-            if isinstance(response_substring, unicode):
+            if isinstance(response_substring, str):
                 response_substring = response_substring.encode("utf-8")
             self.assertIn(response_substring, body)
         returnValue(body)

src/allmydata/test/common_util.py

@@ -20,7 +20,7 @@ from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv
 
 
 def skip_if_cannot_represent_filename(u):
@@ -33,13 +33,6 @@ def skip_if_cannot_represent_filename(u):
         except UnicodeEncodeError:
             raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
 
-def skip_if_cannot_represent_argv(u):
-    precondition(isinstance(u, unicode))
-    try:
-        u.encode(get_io_encoding())
-    except UnicodeEncodeError:
-        raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.")
-
 
 def _getvalue(io):
     """
@@ -395,28 +388,8 @@ class TimezoneMixin(object):
         return hasattr(time, 'tzset')
 
 
-try:
-    import win32file
-    import win32con
-    def make_readonly(path):
-        win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY)
-    def make_accessible(path):
-        win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL)
-except ImportError:
-    import stat
-    def _make_readonly(path):
-        os.chmod(path, stat.S_IREAD)
-        os.chmod(os.path.dirname(path), stat.S_IREAD)
-    def _make_accessible(path):
-        os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD)
-        os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD)
-    make_readonly = _make_readonly
-    make_accessible = _make_accessible
-
-
 __all__ = [
-    "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin",
-    "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli",
+    "TestMixin", "ShouldFailMixin", "StallMixin", "run_cli", "parse_cli",
     "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit",
     "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin"
 ]

src/allmydata/test/common_web.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from six import ensure_str
 
 __all__ = [

src/allmydata/test/status.py

@@ -1,16 +1 @@
 
-class FakeStatus(object):
-    def __init__(self):
-        self.status = []
-
-    def setServiceParent(self, p):
-        pass
-
-    def get_status(self):
-        return self.status
-
-    def get_storage_index(self):
-        return None
-
-    def get_size(self):
-        return None

src/allmydata/test/storage_plugin.py

@@ -1,8 +1,17 @@
 """
 A storage server plugin the test suite can use to validate the
 functionality.
-"""
 
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from future.utils import native_str, native_str_to_bytes
 from six import ensure_str
 

src/allmydata/test/test_python2_regressions.py

@@ -15,7 +15,6 @@ from testtools.matchers import (
 
 BLACKLIST = {
     "allmydata.scripts.types_",
-    "allmydata.test.check_load",
     "allmydata.test._win_subprocess",
     "allmydata.windows.registry",
     "allmydata.windows.fixups",

src/allmydata/test/test_system.py

@@ -50,8 +50,7 @@ from twisted.python.failure import Failure
 from twisted.python.filepath import (
     FilePath,
 )
-
-from ._twisted_9607 import (
+from twisted.internet.utils import (
     getProcessOutputAndValue,
 )
 

src/allmydata/test/web/matchers.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import attr
 
 from testtools.matchers import Mismatch

src/allmydata/util/_python3.py

@@ -98,11 +98,17 @@ PORTED_MODULES = [
     "allmydata.storage.shares",
     "allmydata.test",
     "allmydata.test.cli",
+    "allmydata.test.cli_node_api",
+    "allmydata.test.common",
+    "allmydata.test.common_web",
     "allmydata.test.no_network",
     "allmydata.test.matchers",
     "allmydata.test.mutable",
     "allmydata.test.mutable.util",
+    "allmydata.test.python3_tests",
+    "allmydata.test.storage_plugin",
     "allmydata.test.web",
+    "allmydata.test.web.matchers",
     "allmydata.testing",
     "allmydata.testing.web",
     "allmydata.unknown",