From b981e90de3286bc2d4244bf304a3bd777056d2aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 10:51:44 -0400 Subject: [PATCH 01/14] Tahoe-LAFS now relies on a sufficiently new version of Twisted, which includes this bugfix. --- src/allmydata/test/_twisted_9607.py | 74 ----------------------------- src/allmydata/test/test_system.py | 3 +- 2 files changed, 1 insertion(+), 76 deletions(-) delete mode 100644 src/allmydata/test/_twisted_9607.py diff --git a/src/allmydata/test/_twisted_9607.py b/src/allmydata/test/_twisted_9607.py deleted file mode 100644 index c4e37ef38..000000000 --- a/src/allmydata/test/_twisted_9607.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -A copy of the implementation of Twisted's ``getProcessOutputAndValue`` -with the fix for Twisted #9607 (support for stdinBytes) patched in. -""" - -from __future__ import ( - division, - absolute_import, - print_function, - unicode_literals, -) - -from io import BytesIO - -from twisted.internet import protocol, defer - - -class _EverythingGetter(protocol.ProcessProtocol, object): - - def __init__(self, deferred, stdinBytes=None): - self.deferred = deferred - self.outBuf = BytesIO() - self.errBuf = BytesIO() - self.outReceived = self.outBuf.write - self.errReceived = self.errBuf.write - self.stdinBytes = stdinBytes - - def connectionMade(self): - if self.stdinBytes is not None: - self.transport.writeToChild(0, self.stdinBytes) - # The only compelling reason not to _always_ close stdin here is - # backwards compatibility. - self.transport.closeStdin() - - def processEnded(self, reason): - out = self.outBuf.getvalue() - err = self.errBuf.getvalue() - e = reason.value - code = e.exitCode - if e.signal: - self.deferred.errback((out, err, e.signal)) - else: - self.deferred.callback((out, err, code)) - - - -def _callProtocolWithDeferred(protocol, executable, args, env, path, - reactor=None, protoArgs=()): - if reactor is None: - from twisted.internet import reactor - - d = defer.Deferred() - p = protocol(d, *protoArgs) - reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) - return d - - - -def getProcessOutputAndValue(executable, args=(), env={}, path=None, - reactor=None, stdinBytes=None): - """Spawn a process and returns a Deferred that will be called back with - its output (from stdout and stderr) and it's exit code as (out, err, code) - If a signal is raised, the Deferred will errback with the stdout and - stderr up to that point, along with the signal, as (out, err, signalNum) - """ - return _callProtocolWithDeferred( - _EverythingGetter, - executable, - args, - env, - path, - reactor, - protoArgs=(stdinBytes,), - ) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index ce575ce7a..040104b4c 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -50,8 +50,7 @@ from twisted.python.failure import Failure from twisted.python.filepath import ( FilePath, ) - -from ._twisted_9607 import ( +from twisted.internet.utils import ( getProcessOutputAndValue, ) From eedc8f23cffc89c320cc065a981dacaaee67344e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:00:12 -0400 Subject: [PATCH 02/14] Delete some unused code. --- src/allmydata/test/common_util.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index f62cd34cc..be00522cd 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -33,13 +33,6 @@ def skip_if_cannot_represent_filename(u): except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") -def skip_if_cannot_represent_argv(u): - precondition(isinstance(u, unicode)) - try: - u.encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.") - def _getvalue(io): """ @@ -395,27 +388,8 @@ class TimezoneMixin(object): return hasattr(time, 'tzset') -try: - import win32file - import win32con - def make_readonly(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY) - def make_accessible(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) -except ImportError: - import stat - def _make_readonly(path): - os.chmod(path, stat.S_IREAD) - os.chmod(os.path.dirname(path), stat.S_IREAD) - def _make_accessible(path): - os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - make_readonly = _make_readonly - make_accessible = _make_accessible - - __all__ = [ - "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin", + "TestMixin", "ShouldFailMixin", "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" From 80385aea8ec9a583302f68244b68490817ac7d7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:13:56 -0400 Subject: [PATCH 03/14] Port to Python 3. --- src/allmydata/test/cli_node_api.py | 15 +++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 34d73a199..4e4173924 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "CLINodeAPI", @@ -81,7 +92,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): self._fds = fds def connectionMade(self): - for proto in self._fds.values(): + for proto in list(self._fds.values()): proto.makeConnection(self.transport) def childDataReceived(self, childFD, data): @@ -94,7 +105,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): def processEnded(self, reason): notified = set() - for proto in self._fds.values(): + for proto in list(self._fds.values()): if proto not in notified: proto.connectionLost(reason) notified.add(proto) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f6857cad..1d31725a6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test", "allmydata.test.cli", + "allmydata.test.cli_node_api", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From 84e32882b425786b14590db9e4d5f8900bccd106 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:07:55 -0400 Subject: [PATCH 04/14] These don't belong in tests module. --- {src/allmydata/test => misc/checkers}/check_load.py | 0 {src/allmydata/test => misc/checkers}/check_memory.py | 0 {src/allmydata/test => misc/checkers}/check_speed.py | 0 src/allmydata/test/test_python2_regressions.py | 1 - 4 files changed, 1 deletion(-) rename {src/allmydata/test => misc/checkers}/check_load.py (100%) rename {src/allmydata/test => misc/checkers}/check_memory.py (100%) rename {src/allmydata/test => misc/checkers}/check_speed.py (100%) diff --git a/src/allmydata/test/check_load.py b/misc/checkers/check_load.py similarity index 100% rename from src/allmydata/test/check_load.py rename to misc/checkers/check_load.py diff --git a/src/allmydata/test/check_memory.py b/misc/checkers/check_memory.py similarity index 100% rename from src/allmydata/test/check_memory.py rename to misc/checkers/check_memory.py diff --git a/src/allmydata/test/check_speed.py b/misc/checkers/check_speed.py similarity index 100% rename from src/allmydata/test/check_speed.py rename to misc/checkers/check_speed.py diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index fc9ebe17a..59b16d011 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -15,7 +15,6 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.scripts.types_", - "allmydata.test.check_load", "allmydata.test._win_subprocess", "allmydata.windows.registry", "allmydata.windows.fixups", From a37121f89ca638fa7621675653fc7401375d258e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:34:51 -0400 Subject: [PATCH 05/14] Already ported. --- src/allmydata/util/_python3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1d31725a6..d747e581f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -103,6 +103,7 @@ PORTED_MODULES = [ "allmydata.test.matchers", "allmydata.test.mutable", "allmydata.test.mutable.util", + "allmydata.test.python3_tests", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 625a0abb025617dc502a06b721478b5ed841f975 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:37:32 -0400 Subject: [PATCH 06/14] Port to Python 3. --- src/allmydata/test/common_web.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index ce1670341..bd55a9fe9 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str __all__ = [ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d747e581f..b6d4fc2ff 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From a367d333d943e73f1c4faeabbbb5a63a872a7225 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:04:09 -0400 Subject: [PATCH 07/14] Port to Python 3. --- src/allmydata/test/common.py | 29 +++++++++++++++++++---------- src/allmydata/util/_python3.py | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d7f00554d..2ce34bb1f 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,4 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import chr as byteschr, str as native_str __all__ = [ "SyncTestCase", @@ -15,8 +26,6 @@ __all__ = [ "PIPE", ] -from past.builtins import chr as byteschr, unicode - import sys import os, random, struct import six @@ -106,7 +115,7 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32": +if sys.platform == "win32" and PY2: # Python 2.7 doesn't have good options for launching a process with # non-ASCII in its command line. So use this alternative that does a # better job. However, only use it on Windows because it doesn't work @@ -253,7 +262,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) @@ -264,7 +273,7 @@ class UseNode(object): return "\n".join( " = ".join((key, value)) for (key, value) - in config.items() + in list(config.items()) ) if self.plugin_config is None: @@ -849,17 +858,17 @@ class WebErrorMixin(object): callable=None, *args, **kwargs): # returns a Deferred with the response body if isinstance(substring, bytes): - substring = unicode(substring, "ascii") - if isinstance(response_substring, unicode): + substring = str(substring, "ascii") + if isinstance(response_substring, str): response_substring = response_substring.encode("ascii") - assert substring is None or isinstance(substring, unicode) + assert substring is None or isinstance(substring, str) assert response_substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: - code_string = unicode(f) + code_string = str(f) self.failUnless(substring in code_string, "%s: substring '%s' not in '%s'" % (which, substring, code_string)) @@ -882,7 +891,7 @@ class WebErrorMixin(object): body = yield response.content() self.assertEquals(response.code, code) if response_substring is not None: - if isinstance(response_substring, unicode): + if isinstance(response_substring, str): response_substring = response_substring.encode("utf-8") self.assertIn(response_substring, body) returnValue(body) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b6d4fc2ff..bef854054 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common", "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", From a11b47785fc6b608d6d7d5d5b156a5d02e1930cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:52:42 -0400 Subject: [PATCH 08/14] Port to Python 3. --- src/allmydata/test/storage_plugin.py | 11 ++++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 17ec89078..d3f1ec7c9 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -1,8 +1,17 @@ """ A storage server plugin the test suite can use to validate the functionality. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str, native_str_to_bytes from six import ensure_str diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bef854054..97cfd64f8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -106,6 +106,7 @@ PORTED_MODULES = [ "allmydata.test.mutable", "allmydata.test.mutable.util", "allmydata.test.python3_tests", + "allmydata.test.storage_plugin", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 74e9bdd476d3ca67fca89047562b81b2fe14e197 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:56:42 -0400 Subject: [PATCH 09/14] No point in having separate module. --- src/allmydata/test/cli/test_status.py | 18 +++++++++++++++++- src/allmydata/test/status.py | 15 --------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index a04939429..4488299b2 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -37,10 +37,26 @@ from allmydata.util import jsonbytes as json from ..no_network import GridTestMixin from ..common_web import do_http -from ..status import FakeStatus from .common import CLITestMixin +class FakeStatus(object): + def __init__(self): + self.status = [] + + def setServiceParent(self, p): + pass + + def get_status(self): + return self.status + + def get_storage_index(self): + return None + + def get_size(self): + return None + + class ProgressBar(unittest.TestCase): def test_ascii0(self): diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py index 44f2123f9..8b1378917 100644 --- a/src/allmydata/test/status.py +++ b/src/allmydata/test/status.py @@ -1,16 +1 @@ -class FakeStatus(object): - def __init__(self): - self.status = [] - - def setServiceParent(self, p): - pass - - def get_status(self): - return self.status - - def get_storage_index(self): - return None - - def get_size(self): - return None From 3bc9b0d5440f4b08506ba7723f3d7aff4da8a2dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:00:27 -0400 Subject: [PATCH 10/14] Port to Python 3. --- src/allmydata/test/web/matchers.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index 99c91ef5c..f764da79d 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import attr from testtools.matchers import Mismatch diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 97cfd64f8..03eafdea5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -108,6 +108,7 @@ PORTED_MODULES = [ "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.web", + "allmydata.test.web.matchers", "allmydata.testing", "allmydata.testing.web", "allmydata.unknown", From 57aa798814bcce22a7651cbdeb8217800db6571a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:01:23 -0400 Subject: [PATCH 11/14] Delete another item that was deleted. --- src/allmydata/test/common_util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index be00522cd..07c4a0ed7 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -389,8 +389,7 @@ class TimezoneMixin(object): __all__ = [ - "TestMixin", "ShouldFailMixin", - "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", + "TestMixin", "ShouldFailMixin", "StallMixin", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" ] From 7b5cb13417cab0ff70d28314ad392fe437ca2eb5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:02:12 -0400 Subject: [PATCH 12/14] Flake fix. --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 07c4a0ed7..91f0b0f78 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -20,7 +20,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv def skip_if_cannot_represent_filename(u): From 8056b43df6ac7a247f9abefb6d6eed15e3af8c81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:21:14 -0400 Subject: [PATCH 13/14] News file. --- newsfragments/3667.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3667.minor diff --git a/newsfragments/3667.minor b/newsfragments/3667.minor new file mode 100644 index 000000000..e69de29bb From 3841662ee6a4a796277b3481ea859b3d07255c90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:23:31 -0400 Subject: [PATCH 14/14] Fix tests on Python 3. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 2ce34bb1f..d874d07ae 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -6,10 +6,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, native_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from past.builtins import chr as byteschr, str as native_str +from past.builtins import chr as byteschr __all__ = [ "SyncTestCase",