From 9ecf9f120f70d060ce70422d580aa0dc5c5f2871 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 22:21:59 -0400 Subject: [PATCH 01/24] Let's start here, for once --- newsfragments/3456.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3456.minor diff --git a/newsfragments/3456.minor b/newsfragments/3456.minor new file mode 100644 index 000000000..e69de29bb From 2c2b61676c28a1d56ff373b927be1802b68fd221 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 22:27:24 -0400 Subject: [PATCH 02/24] Move LoggingServiceParent back to common --- src/allmydata/test/common.py | 8 +++++++- src/allmydata/test/common_py3.py | 7 ------- src/allmydata/test/test_storage.py | 3 ++- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 6b0059cb4..ca0bc5397 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -46,6 +46,7 @@ from testtools.twistedsupport import ( flush_logged_errors, ) +from twisted.application import service from twisted.plugin import IPlugin from twisted.internet import defer from twisted.internet.defer import inlineCallbacks, returnValue @@ -88,7 +89,7 @@ from .eliotutil import ( EliotLoggedRunTest, ) # Backwards compatibility imports: -from .common_py3 import LoggingServiceParent, ShouldFailMixin # noqa: F401 +from .common_py3 import ShouldFailMixin # noqa: F401 TEST_RSA_KEY_SIZE = 522 @@ -781,6 +782,11 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None): return filenode +class LoggingServiceParent(service.MultiService): + def log(self, *args, **kwargs): + return log.msg(*args, **kwargs) + + TEST_DATA=b"\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index c3a84189b..65b79935a 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -19,13 +19,11 @@ import time import signal from twisted.internet import defer, reactor -from twisted.application import service from twisted.python import failure from twisted.trial import unittest from ..util.assertutil import precondition from ..util.encodingutil import unicode_platform, get_filesystem_encoding -from ..util import log class TimezoneMixin(object): @@ -163,8 +161,3 @@ class FakeCanary(object): return None def getPeer(self): return "" - - -class LoggingServiceParent(service.MultiService): - def log(self, *args, **kwargs): - return log.msg(*args, **kwargs) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 30abf9cdc..8f8129ef2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -51,7 +51,8 @@ from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, ) -from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin +from .common import LoggingServiceParent +from .common_py3 import FakeCanary, ShouldFailMixin class UtilTests(unittest.TestCase): From 23140b8b1c5a75f70833443bc764719cf7d2df2f Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 22:55:33 -0400 Subject: [PATCH 03/24] Move ShouldFailMixin back to common_util There were originally two versions of this, one in common and another in common_util. We moved both into common_py3 but then removed the one from common, so here we move back to common_util, while allowing imports from common to avoid a noisy changeset. --- src/allmydata/test/common.py | 3 +- src/allmydata/test/common_py3.py | 48 +-------------------------- src/allmydata/test/common_util.py | 49 +++++++++++++++++++++++++++- src/allmydata/test/test_happiness.py | 2 +- src/allmydata/test/test_storage.py | 4 +-- src/allmydata/test/test_upload.py | 2 +- 6 files changed, 54 insertions(+), 54 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index ca0bc5397..90e97f11a 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -88,8 +88,7 @@ from ..crypto import ( from .eliotutil import ( EliotLoggedRunTest, ) -# Backwards compatibility imports: -from .common_py3 import ShouldFailMixin # noqa: F401 +from .common_util import ShouldFailMixin # noqa: F401 TEST_RSA_KEY_SIZE = 522 diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 65b79935a..f0c4af6ba 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -18,8 +18,7 @@ import os import time import signal -from twisted.internet import defer, reactor -from twisted.python import failure +from twisted.internet import reactor from twisted.trial import unittest from ..util.assertutil import precondition @@ -73,51 +72,6 @@ class SignalMixin(object): return super(SignalMixin, self).tearDown() -class ShouldFailMixin(object): - - def shouldFail(self, expected_failure, which, substring, - callable, *args, **kwargs): - """Assert that a function call raises some exception. This is a - Deferred-friendly version of TestCase.assertRaises() . - - Suppose you want to verify the following function: - - def broken(a, b, c): - if a < 0: - raise TypeError('a must not be negative') - return defer.succeed(b+c) - - You can use: - d = self.shouldFail(TypeError, 'test name', - 'a must not be negative', - broken, -4, 5, c=12) - in your test method. The 'test name' string will be included in the - error message, if any, because Deferred chains frequently make it - difficult to tell which assertion was tripped. - - The substring= argument, if not None, must appear in the 'repr' - of the message wrapped by this Failure, or the test will fail. - """ - - assert substring is None or isinstance(substring, (bytes, unicode)) - d = defer.maybeDeferred(callable, *args, **kwargs) - def done(res): - if isinstance(res, failure.Failure): - res.trap(expected_failure) - if substring: - self.failUnless(substring in str(res), - "%s: substring '%s' not in '%s'" - % (which, substring, str(res))) - # return the Failure for further analysis, but in a form that - # doesn't make the Deferred chain think that we failed. - return [res] - else: - self.fail("%s was supposed to raise %s, not get '%s'" % - (which, expected_failure, res)) - d.addBoth(done) - return d - - class ReallyEqualMixin(object): def failUnlessReallyEqual(self, a, b, msg=None): self.assertEqual(a, b, msg) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 6b964368d..6a0ee09dc 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -5,6 +5,7 @@ from random import randrange from six.moves import StringIO from twisted.internet import reactor, defer +from twisted.python import failure from twisted.trial import unittest from ..util.assertutil import precondition @@ -12,8 +13,9 @@ from ..scripts import runner from allmydata.util.encodingutil import get_io_encoding # Imported for backwards compatibility: from future.utils import bord, bchr, binary_type +from past.builtins import unicode from .common_py3 import ( - SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, ShouldFailMixin + SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin ) @@ -85,6 +87,51 @@ class StallMixin(object): return d +class ShouldFailMixin(object): + + def shouldFail(self, expected_failure, which, substring, + callable, *args, **kwargs): + """Assert that a function call raises some exception. This is a + Deferred-friendly version of TestCase.assertRaises() . + + Suppose you want to verify the following function: + + def broken(a, b, c): + if a < 0: + raise TypeError('a must not be negative') + return defer.succeed(b+c) + + You can use: + d = self.shouldFail(TypeError, 'test name', + 'a must not be negative', + broken, -4, 5, c=12) + in your test method. The 'test name' string will be included in the + error message, if any, because Deferred chains frequently make it + difficult to tell which assertion was tripped. + + The substring= argument, if not None, must appear in the 'repr' + of the message wrapped by this Failure, or the test will fail. + """ + + assert substring is None or isinstance(substring, (bytes, unicode)) + d = defer.maybeDeferred(callable, *args, **kwargs) + def done(res): + if isinstance(res, failure.Failure): + res.trap(expected_failure) + if substring: + self.failUnless(substring in str(res), + "%s: substring '%s' not in '%s'" + % (which, substring, str(res))) + # return the Failure for further analysis, but in a form that + # doesn't make the Deferred chain think that we failed. + return [res] + else: + self.fail("%s was supposed to raise %s, not get '%s'" % + (which, expected_failure, res)) + d.addBoth(done) + return d + + class TestMixin(SignalMixin): def setUp(self): return super(TestMixin, self).setUp() diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index 021b75d33..9ff36ef26 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -23,7 +23,7 @@ from hypothesis.strategies import text, sets from allmydata.immutable import happiness_upload from allmydata.util.happinessutil import servers_of_happiness, \ shares_by_server, merge_servers -from allmydata.test.common_py3 import ShouldFailMixin +from allmydata.test.common import ShouldFailMixin class HappinessUploadUtils(unittest.TestCase): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8f8129ef2..83fcd6f28 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -51,8 +51,8 @@ from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, ) -from .common import LoggingServiceParent -from .common_py3 import FakeCanary, ShouldFailMixin +from .common import LoggingServiceParent, ShouldFailMixin +from .common_py3 import FakeCanary class UtilTests(unittest.TestCase): diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 5ce2a29fb..a6438c1fc 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -28,12 +28,12 @@ from allmydata.util import log, base32 from allmydata.util.assertutil import precondition from allmydata.util.deferredutil import DeferredListShouldSucceed from allmydata.test.no_network import GridTestMixin -from allmydata.test.common_py3 import ShouldFailMixin from allmydata.storage_client import StorageFarmBroker from allmydata.storage.server import storage_index_to_dir from allmydata.client import _Client from .common import ( EMPTY_CLIENT_CONFIG, + ShouldFailMixin, ) from functools import reduce From b75b48e68c2f906dfaf295cce4ad187906e49be9 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 23:05:52 -0400 Subject: [PATCH 04/24] Move FakeCanary to common_util Couldn't bring myself to move it back to test_storage. --- src/allmydata/test/common_py3.py | 28 ----------------------- src/allmydata/test/common_util.py | 28 +++++++++++++++++++++++ src/allmydata/test/test_crawler.py | 3 +-- src/allmydata/test/test_storage.py | 2 +- src/allmydata/test/test_storage_web.py | 2 +- src/allmydata/test/web/test_introducer.py | 2 +- 6 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index f0c4af6ba..f917cf8fb 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -87,31 +87,3 @@ def skip_if_cannot_represent_filename(u): u.encode(enc) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") - - -class Marker(object): - pass - -class FakeCanary(object): - """For use in storage tests. - - Can be moved back to test_storage.py once enough Python 3 porting has been - done. - """ - def __init__(self, ignore_disconnectors=False): - self.ignore = ignore_disconnectors - self.disconnectors = {} - def notifyOnDisconnect(self, f, *args, **kwargs): - if self.ignore: - return - m = Marker() - self.disconnectors[m] = (f, args, kwargs) - return m - def dontNotifyOnDisconnect(self, marker): - if self.ignore: - return - del self.disconnectors[marker] - def getRemoteTubID(self): - return None - def getPeer(self): - return "" diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 6a0ee09dc..3b81be109 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -87,6 +87,34 @@ class StallMixin(object): return d +class Marker(object): + pass + +class FakeCanary(object): + """For use in storage tests. + + Can be moved back to test_storage.py once enough Python 3 porting has been + done. + """ + def __init__(self, ignore_disconnectors=False): + self.ignore = ignore_disconnectors + self.disconnectors = {} + def notifyOnDisconnect(self, f, *args, **kwargs): + if self.ignore: + return + m = Marker() + self.disconnectors[m] = (f, args, kwargs) + return m + def dontNotifyOnDisconnect(self, marker): + if self.ignore: + return + del self.disconnectors[marker] + def getRemoteTubID(self): + return None + def getPeer(self): + return "" + + class ShouldFailMixin(object): def shouldFail(self, expected_failure, which, substring, diff --git a/src/allmydata/test/test_crawler.py b/src/allmydata/test/test_crawler.py index 1ed217251..a9be90c43 100644 --- a/src/allmydata/test/test_crawler.py +++ b/src/allmydata/test/test_crawler.py @@ -27,8 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin from allmydata.storage.server import StorageServer, si_b2a from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded -from allmydata.test.common_py3 import FakeCanary -from allmydata.test.common_util import StallMixin +from allmydata.test.common_util import StallMixin, FakeCanary class BucketEnumeratingCrawler(ShareCrawler): cpu_slice = 500 # make sure it can complete in a single slice diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 83fcd6f28..01ae10280 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -52,7 +52,7 @@ from allmydata.storage_client import ( _StorageServer, ) from .common import LoggingServiceParent, ShouldFailMixin -from .common_py3 import FakeCanary +from .common_util import FakeCanary class UtilTests(unittest.TestCase): diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index 19f98851f..aa1f19936 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -50,7 +50,7 @@ from allmydata.web.storage import ( StorageStatusElement, remove_prefix ) -from .common_py3 import FakeCanary +from .common_util import FakeCanary def remove_tags(s): s = re.sub(br'<[^>]*>', b' ', s) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 5fdff47ad..bf6ef6a4b 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -33,7 +33,7 @@ from .common import ( from ..common import ( SameProcessStreamEndpointAssigner, ) -from ..common_py3 import ( +from ..common_util import ( FakeCanary, ) from ..common_web import ( From 93d4a8373f058b7005930ba62262d2748956c4f3 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 23:14:32 -0400 Subject: [PATCH 05/24] Move ReallyEqualMixin and s31e back to common_util s31e = skip_if_cannot_represent_filename --- src/allmydata/test/common_py3.py | 22 ---------------------- src/allmydata/test/common_util.py | 22 ++++++++++++++++++---- src/allmydata/test/test_encodingutil.py | 2 +- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index f917cf8fb..0dae05aa6 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -12,17 +12,12 @@ from __future__ import print_function from future.utils import PY2 if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from past.builtins import unicode import os import time import signal from twisted.internet import reactor -from twisted.trial import unittest - -from ..util.assertutil import precondition -from ..util.encodingutil import unicode_platform, get_filesystem_encoding class TimezoneMixin(object): @@ -70,20 +65,3 @@ class SignalMixin(object): if self.sigchldHandler: signal.signal(signal.SIGCHLD, self.sigchldHandler) return super(SignalMixin, self).tearDown() - - -class ReallyEqualMixin(object): - def failUnlessReallyEqual(self, a, b, msg=None): - self.assertEqual(a, b, msg) - self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) - - -def skip_if_cannot_represent_filename(u): - precondition(isinstance(u, unicode)) - - enc = get_filesystem_encoding() - if not unicode_platform(): - try: - u.encode(enc) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 3b81be109..ce6271399 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -10,15 +10,23 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import get_io_encoding +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding # Imported for backwards compatibility: from future.utils import bord, bchr, binary_type from past.builtins import unicode -from .common_py3 import ( - SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin -) +from .common_py3 import SignalMixin +def skip_if_cannot_represent_filename(u): + precondition(isinstance(u, unicode)) + + enc = get_filesystem_encoding() + if not unicode_platform(): + try: + u.encode(enc) + except UnicodeEncodeError: + raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") + def skip_if_cannot_represent_argv(u): precondition(isinstance(u, unicode)) try: @@ -80,6 +88,12 @@ def flip_one_bit(s, offset=0, size=None): return result +class ReallyEqualMixin(object): + def failUnlessReallyEqual(self, a, b, msg=None): + self.assertEqual(a, b, msg) + self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) + + class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index 376bd6ec6..cbc9143b7 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -77,7 +77,7 @@ from twisted.trial import unittest from twisted.python.filepath import FilePath -from allmydata.test.common_py3 import ( +from allmydata.test.common_util import ( ReallyEqualMixin, skip_if_cannot_represent_filename, ) from allmydata.util import encodingutil, fileutil From 41fcd9673e4bea0e69f8e1c446af60760bf03c14 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 30 Sep 2020 23:26:31 -0400 Subject: [PATCH 06/24] Move SignalMixin and TimeMixin back to common_util This concludes our service. --- src/allmydata/test/common_py3.py | 67 -------------------------- src/allmydata/test/common_util.py | 50 ++++++++++++++++++- src/allmydata/test/test_iputil.py | 2 +- src/allmydata/test/test_time_format.py | 2 +- src/allmydata/test/web/test_web.py | 2 +- src/allmydata/util/_python3.py | 1 - 6 files changed, 52 insertions(+), 72 deletions(-) delete mode 100644 src/allmydata/test/common_py3.py diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py deleted file mode 100644 index 0dae05aa6..000000000 --- a/src/allmydata/test/common_py3.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Common utilities that have been ported to Python 3. - -Ported to Python 3. -""" - -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import os -import time -import signal - -from twisted.internet import reactor - - -class TimezoneMixin(object): - - def setTimezone(self, timezone): - def tzset_if_possible(): - # Windows doesn't have time.tzset(). - if hasattr(time, 'tzset'): - time.tzset() - - unset = object() - originalTimezone = os.environ.get('TZ', unset) - def restoreTimezone(): - if originalTimezone is unset: - del os.environ['TZ'] - else: - os.environ['TZ'] = originalTimezone - tzset_if_possible() - - os.environ['TZ'] = timezone - self.addCleanup(restoreTimezone) - tzset_if_possible() - - def have_working_tzset(self): - return hasattr(time, 'tzset') - - -class SignalMixin(object): - # This class is necessary for any code which wants to use Processes - # outside the usual reactor.run() environment. It is copied from - # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses - # something rather different. - sigchldHandler = None - - def setUp(self): - # make sure SIGCHLD handler is installed, as it should be on - # reactor.run(). problem is reactor may not have been run when this - # test runs. - if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): - self.sigchldHandler = signal.signal(signal.SIGCHLD, - reactor._handleSigchld) - return super(SignalMixin, self).setUp() - - def tearDown(self): - if self.sigchldHandler: - signal.signal(signal.SIGCHLD, self.sigchldHandler) - return super(SignalMixin, self).tearDown() diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index ce6271399..8673e2ece 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,8 @@ from __future__ import print_function import os +import time +import signal from random import randrange from six.moves import StringIO @@ -14,7 +16,6 @@ from allmydata.util.encodingutil import unicode_platform, get_filesystem_encodin # Imported for backwards compatibility: from future.utils import bord, bchr, binary_type from past.builtins import unicode -from .common_py3 import SignalMixin def skip_if_cannot_represent_filename(u): @@ -94,6 +95,28 @@ class ReallyEqualMixin(object): self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) +class SignalMixin(object): + # This class is necessary for any code which wants to use Processes + # outside the usual reactor.run() environment. It is copied from + # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses + # something rather different. + sigchldHandler = None + + def setUp(self): + # make sure SIGCHLD handler is installed, as it should be on + # reactor.run(). problem is reactor may not have been run when this + # test runs. + if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"): + self.sigchldHandler = signal.signal(signal.SIGCHLD, + reactor._handleSigchld) + return super(SignalMixin, self).setUp() + + def tearDown(self): + if self.sigchldHandler: + signal.signal(signal.SIGCHLD, self.sigchldHandler) + return super(SignalMixin, self).tearDown() + + class StallMixin(object): def stall(self, res=None, delay=1): d = defer.Deferred() @@ -221,6 +244,31 @@ class TestMixin(SignalMixin): self.fail("Reactor was still active when it was required to be quiescent.") +class TimezoneMixin(object): + + def setTimezone(self, timezone): + def tzset_if_possible(): + # Windows doesn't have time.tzset(). + if hasattr(time, 'tzset'): + time.tzset() + + unset = object() + originalTimezone = os.environ.get('TZ', unset) + def restoreTimezone(): + if originalTimezone is unset: + del os.environ['TZ'] + else: + os.environ['TZ'] = originalTimezone + tzset_if_possible() + + os.environ['TZ'] = timezone + self.addCleanup(restoreTimezone) + tzset_if_possible() + + def have_working_tzset(self): + return hasattr(time, 'tzset') + + try: import win32file import win32con diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 71374fec7..f403de35b 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -23,7 +23,7 @@ from tenacity import retry, stop_after_attempt from foolscap.api import Tub from allmydata.util import iputil, gcutil -import allmydata.test.common_py3 as testutil +import allmydata.test.common_util as testutil from allmydata.util.namespace import Namespace diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index dc9c03b91..f83a6a53c 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -16,7 +16,7 @@ import time from twisted.trial import unittest -from allmydata.test.common_py3 import TimezoneMixin +from allmydata.test.common_util import TimezoneMixin from allmydata.util import time_format diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1773c32a4..508fc82d4 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -52,7 +52,7 @@ from allmydata.interfaces import ( ) from allmydata.mutable import servermap, publish, retrieve from .. import common_util as testutil -from ..common_py3 import TimezoneMixin +from ..common_util import TimezoneMixin from ..common_web import ( do_http, Error, diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 31ac4f4ae..93e0ad7e8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -52,7 +52,6 @@ PORTED_MODULES = [ "allmydata.storage.mutable", "allmydata.storage.server", "allmydata.storage.shares", - "allmydata.test.common_py3", "allmydata.test.no_network", "allmydata.uri", "allmydata.util._python3", From 5899cfdabdcc2e108d510588922586bf9f90c3d9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Oct 2020 10:48:09 -0400 Subject: [PATCH 07/24] Fix callRemote unicode issues on Python 2 universally, using monkeypatching. --- newsfragments/3458.minor | 0 src/allmydata/__init__.py | 6 +++ src/allmydata/_monkeypatch.py | 41 +++++++++++++++++++++ src/allmydata/immutable/downloader/share.py | 6 +-- src/allmydata/util/_python3.py | 1 + 5 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 newsfragments/3458.minor create mode 100644 src/allmydata/_monkeypatch.py diff --git a/newsfragments/3458.minor b/newsfragments/3458.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index bbfcb89d4..15d5fb240 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -42,3 +42,9 @@ __full_version__ = __appname__ + '/' + str(__version__) # Install Python 3 module locations in Python 2: from future import standard_library standard_library.install_aliases() + + +# Monkey-patch 3rd party libraries: +from ._monkeypatch import patch +patch() +del patch diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py new file mode 100644 index 000000000..87475f7de --- /dev/null +++ b/src/allmydata/_monkeypatch.py @@ -0,0 +1,41 @@ +""" +Monkey-patching of third party libraries. + +Ported to Python 3. +""" + + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + + +def patch(): + """Path third-party libraries to make Tahoe-LAFS work.""" + # Make sure Foolscap always get native strings passed to method names in callRemote. + # This can be removed when any one of the following happens: + # + # 1. Tahoe-LAFS on Python 2 switches to version of Foolscap that fixes + # https://github.com/warner/foolscap/issues/72 + # 2. Foolscap is dropped as a dependency. + # 3. Tahoe-LAFS drops Python 2 support. + + if PY2: + # Only tested with this version; ensure correctness with new releases, + # and then either update the assert or hopefully drop the monkeypatch. + from foolscap import __version__ + assert __version__ == "0.13.1", "Wrong version %s of Foolscap" % (__version__,) + + from foolscap.referenceable import RemoteReference + original_getMethodInfo = RemoteReference._getMethodInfo + + def _getMethodInfo(self, name): + if isinstance(name, str): + name = name.encode("utf-8") + return original_getMethodInfo(self, name) + RemoteReference._getMethodInfo = _getMethodInfo diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 261ed79e1..f279018ba 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -6,7 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, native_str +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -765,9 +765,7 @@ class Share(object): level=log.WEIRD, umid="qZu0wg")) def _send_request(self, start, length): - # For some reason tests fail on Python 2 if this is not a native - # string... - return self._rref.callRemote(native_str("read"), start, length) + return self._rref.callRemote("read", start, length) def _got_data(self, data, start, length, block_ev, lp): block_ev.finished(len(data), now()) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index ccff0958a..8d00afc2c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -24,6 +24,7 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata._monkeypatch", "allmydata.codec", "allmydata.crypto", "allmydata.crypto.aes", From 3f297bf0e3c2abf3ff5aac983d83f0427e427d3b Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Fri, 2 Oct 2020 09:56:55 -0700 Subject: [PATCH 08/24] feat(py3): Fix test runner exception The exception this addresses was preventing the test runner from running to completion under Python 3, thus preventing using the full test suite to evaluate the impact of porting efforts. This change causes no regressions in the test suite under python 2.7 and only affects tests AFAICT. I added debug logging to check all the strings that seem to make it to that point and I didn't see anything that looked like it needed to be decoded in particular, so I think this change is relatively safe. The traceback for reference ``` Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/bin/trial", line 8, in sys.exit(run()) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/scripts/trial.py", line 621, in run test_result = trialRunner.run(suite) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 998, in run return self._runWithoutDecoration(test, self._forceGarbageCollection) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 1025, in _runWithoutDecoration run() File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 1020, in run = lambda: suite.run(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 253, in run TestSuite.run(self, result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/_asyncrunner.py", line 38, in run test(result) File "/usr/lib/python3.6/unittest/suite.py", line 84, in __call__ return self.run(*args, **kwds) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 212, in run super(LoggedSuite, self).run(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/_asyncrunner.py", line 38, in run test(result) File "/usr/lib/python3.6/unittest/suite.py", line 84, in __call__ return self.run(*args, **kwds) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 185, in run test(result) File "/usr/lib/python3.6/unittest/suite.py", line 84, in __call__ return self.run(*args, **kwds) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 185, in run test(result) File "/usr/lib/python3.6/unittest/suite.py", line 84, in __call__ return self.run(*args, **kwds) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/runner.py", line 185, in run test(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/_asyncrunner.py", line 59, in __call__ return self.run(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/twisted/trial/_asyncrunner.py", line 69, in run reporter._AdaptedReporter(result, self.__class__)) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36/lib/python3.6/site-packages/testtools/testcase.py", line 675, in run return run_test.run(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/eliotutil.py", line 110, in run_and_republish with RUN_TEST(name=self.id().decode("utf-8")).context() as action: AttributeError: 'str' object has no attribute 'decode' ``` --- src/allmydata/test/eliotutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 3fbf3001c..21e109164 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -106,7 +106,7 @@ def eliot_logged_test(f): # Begin an action that should comprise all messages from the decorated # test method. - with RUN_TEST(name=self.id().decode("utf-8")).context() as action: + with RUN_TEST(name=self.id()).context() as action: # When the test method Deferred fires, the RUN_TEST action is # done. However, we won't have re-published the MemoryLogger # messages into the global/default logger when this Deferred From 447881a0e06b60cac8c5479512740fbef3d2b298 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Wed, 30 Sep 2020 09:56:43 -0700 Subject: [PATCH 09/24] feat(py3): Cleanup redundant string cast I confirmed that `add_version(...)` itself calls `str(...)` on the argument that `things_version` is passed in to under both the Python 2.7 and Python 3.6 version of the library so this is unnecessary here. This results in an empty diff in py3 tests output. --- newsfragments/3455.minor | 1 + src/allmydata/node.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3455.minor diff --git a/newsfragments/3455.minor b/newsfragments/3455.minor new file mode 100644 index 000000000..fc3ba420a --- /dev/null +++ b/newsfragments/3455.minor @@ -0,0 +1 @@ +Cleanup casting to string for better Python 3 compatibility. diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 101482382..9ba4b0d37 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -70,7 +70,7 @@ def _common_valid_config(): # Add our application versions to the data that Foolscap's LogPublisher # reports. for thing, things_version in get_package_versions().items(): - app_versions.add_version(thing, str(things_version)) + app_versions.add_version(thing, things_version) # group 1 will be addr (dotted quad string), group 3 if any will be portnum (string) ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$") From b2332b5bf1e58b879b296e41fcdb3e80c9c07f78 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Thu, 1 Oct 2020 08:40:17 -0700 Subject: [PATCH 10/24] fix(py3): Duplicate section name that py3 catches Python 3 raises an exception that Python 2 doesn't and as such exposes malformed cfg/ini contents used in the tests. The test output diff shows that this mostly converts test `DuplicateSectionError` errors to test errors with different exceptions which is reasonable for the type of fix this is. It does seem to reduce coverage which I'm guessing is because the malformed contents were triggering error handling code paths that aren't triggered now, but I haven't confirmed that. I would think that to cover those cases we should write tests that do that explicitly rather than accidentally. ```diff --- ../../.tox/make-test-py3-all-old.log 2020-10-04 15:13:09.670578482 -0700 +++ ../../.tox/make-test-py3-all-new.log 2020-10-04 15:16:34.054975263 -0700 @@ -1835,7 +1835,7 @@ raise self.failureException(msg) twisted.trial.unittest.FailTest: ['allmydata', 'allmydata.__main__', 'allm[5873 chars]try'] != set() : Some unported modules remain: Ported files: 96 / 292 -Ported lines: 27978 / 93480 +Ported lines: 27978 / 93482 allmydata.test.test_python3.Python3PortingEffortTests.test_finished_porting @@ -2166,11 +2166,11 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 685, in test_disabled_but_helper + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 687, in test_disabled_but_helper yield client.create_client(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ self._expectedName, reason.getTraceback()), -twisted.trial.unittest.FailTest: configparser.DuplicateSectionError raised instead of ValueError: +twisted.trial.unittest.FailTest: builtins.NameError raised instead of ValueError: Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) @@ -2178,12 +2178,12 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 685, in test_disabled_but_helper + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 687, in test_disabled_but_helper yield client.create_client(basedir) --- --- - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 685, in test_disabled_but_helper + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 687, in test_disabled_but_helper yield client.create_client(basedir) -configparser.DuplicateSectionError: While reading from '/home/rpatterson/src/work/sfu/tahoe-lafs/_trial_temp/test_node/test_disabled_but_helper/tahoe.cfg' [line 10]: section 'node' already exists +builtins.NameError: name 'unicode' is not defined allmydata.test.test_node.ClientNotListening.test_disabled_but_helper @@ -2194,11 +2194,11 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 668, in test_disabled_but_storage + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 670, in test_disabled_but_storage yield client.create_client(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ self._expectedName, reason.getTraceback()), -twisted.trial.unittest.FailTest: configparser.DuplicateSectionError raised instead of ValueError: +twisted.trial.unittest.FailTest: builtins.NameError raised instead of ValueError: Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) @@ -2206,12 +2206,12 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 668, in test_disabled_but_storage + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 670, in test_disabled_but_storage yield client.create_client(basedir) --- --- - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 668, in test_disabled_but_storage + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 670, in test_disabled_but_storage yield client.create_client(basedir) -configparser.DuplicateSectionError: While reading from '/home/rpatterson/src/work/sfu/tahoe-lafs/_trial_temp/test_node/test_disabled_but_storage/tahoe.cfg' [line 10]: section 'node' already exists +builtins.NameError: name 'unicode' is not defined allmydata.test.test_node.ClientNotListening.test_disabled_but_storage @@ -2222,7 +2222,7 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 739, in test_create_client_invalid_config + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config yield client.create_client(self.basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ self._expectedName, reason.getTraceback()), @@ -2234,10 +2234,10 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 739, in test_create_client_invalid_config + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config yield client.create_client(self.basedir) --- --- - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 739, in test_create_client_invalid_config + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config yield client.create_client(self.basedir) builtins.TypeError: startswith first arg must be str or a tuple of str, not bytes @@ -2250,7 +2250,7 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 702, in test_port_none_introducer + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 704, in test_port_none_introducer yield create_introducer(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ self._expectedName, reason.getTraceback()), @@ -2262,10 +2262,10 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 702, in test_port_none_introducer + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 704, in test_port_none_introducer yield create_introducer(basedir) --- --- - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 702, in test_port_none_introducer + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 704, in test_port_none_introducer yield create_introducer(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 76, in create_introducer tor_provider, @@ -7432,23 +7432,17 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 655, in test_disabled + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 657, in test_disabled n = yield client.create_client(basedir) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 243, in create_client - config = read_config(basedir, u"client.port") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 219, in read_config - _valid_config=_valid_config(), - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 187, in read_config - parser = configutil.get_config(config_fname) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/configutil.py", line 50, in get_config - config.readfp(f) - File "/usr/lib/python3.6/configparser.py", line 764, in readfp - self.read_file(fp, source=filename) - File "/usr/lib/python3.6/configparser.py", line 718, in read_file - self._read(f, source) - File "/usr/lib/python3.6/configparser.py", line 1067, in _read - lineno) -configparser.DuplicateSectionError: While reading from '/home/rpatterson/src/work/sfu/tahoe-lafs/_trial_temp/test_node/test_disabled/tahoe.cfg' [line 10]: section 'node' already exists + File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks + result = g.send(result) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 285, in create_client_from_config + introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 519, in create_introducer_clients + introducer_cache_filepath, + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/client.py", line 29, in __init__ + assert type(nickname) is unicode +builtins.NameError: name 'unicode' is not defined allmydata.test.test_node.ClientNotListening.test_disabled =============================================================================== @@ -7456,7 +7450,7 @@ Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 577, in test_listen_on_zero t = FakeTub() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 549, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 548, in __init__ self.tubID = base64.b32encode("foo") File "/usr/lib/python3.6/base64.py", line 154, in b32encode s = memoryview(s).tobytes() @@ -7466,9 +7460,9 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 605, in test_multiple_ports + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 606, in test_multiple_ports t = FakeTub() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 549, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 548, in __init__ self.tubID = base64.b32encode("foo") File "/usr/lib/python3.6/base64.py", line 154, in b32encode s = memoryview(s).tobytes() @@ -7478,9 +7472,9 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 624, in test_tor_i2p_listeners + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 626, in test_tor_i2p_listeners t = FakeTub() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 549, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 548, in __init__ self.tubID = base64.b32encode("foo") File "/usr/lib/python3.6/base64.py", line 154, in b32encode s = memoryview(s).tobytes() @@ -8781,11 +8775,11 @@ src/allmydata/immutable/downloader/__init__.py 7 1 2 1 78% 12, 11->12 src/allmydata/immutable/downloader/common.py 14 1 2 1 88% 12, 11->12 src/allmydata/immutable/downloader/fetcher.py 147 1 56 1 99% 12, 11->12 -src/allmydata/immutable/downloader/finder.py 143 9 40 5 92% 11, 88-89, 108, 115, 170-173, 10->11, 87->88, 107->108, 113->115, 165->exit +src/allmydata/immutable/downloader/finder.py 143 11 40 5 91% 11, 88-89, 108, 115, 170-173, 231-232, 10->11, 87->88, 107->108, 113->115, 165->exit src/allmydata/immutable/downloader/node.py 282 9 66 9 94% 11, 170-172, 224-235, 271, 10->11, 43->exit, 60->68, 165->169, 169->170, 256->exit, 270->271, 517->exit, 537->exit src/allmydata/immutable/downloader/segmentation.py 118 1 22 1 99% 11, 10->11 -src/allmydata/immutable/downloader/share.py 453 15 154 8 96% 11, 210, 248-250, 354, 430-437, 518, 722-725, 862, 10->11, 209->210, 239->248, 352->354, 515->518, 660->663, 710->722, 762->exit -src/allmydata/immutable/downloader/status.py 154 9 36 1 95% 11, 223, 226, 230, 232, 234, 274, 285, 287, 10->11 +src/allmydata/immutable/downloader/share.py 453 23 154 11 94% 11, 210, 239-250, 289-290, 354, 399-400, 430-437, 518, 722-725, 775, 862, 10->11, 209->210, 222->239, 288->289, 352->354, 397->399, 515->518, 660->663, 710->722, 762->exit, 774->775 +src/allmydata/immutable/downloader/status.py 154 12 36 1 93% 11, 65-67, 223, 226, 230, 232, 234, 274, 285, 287, 10->11 src/allmydata/immutable/encode.py 421 13 124 10 95% 114, 197-200, 278-280, 405, 412, 462, 509, 562, 687, 102->104, 113->114, 195->197, 404->405, 411->412, 461->462, 499->509, 505->511, 561->562, 685->687 src/allmydata/immutable/filenode.py 196 48 30 5 72% 77-78, 83, 85, 88, 94-101, 104-124, 127-172, 254, 258, 315, 40->42, 224->227, 227->229, 251->254, 257->258 src/allmydata/immutable/happiness_upload.py 214 1 132 3 99% 15, 13->15, 213->211, 280->279 @@ -8808,7 +8802,7 @@ src/allmydata/mutable/repairer.py 57 37 18 0 29% 13, 15, 17, 19, 29-34, 65-71, 74-126, 129-131 src/allmydata/mutable/retrieve.py 489 123 120 33 71% 46, 48, 50, 52, 56, 58, 60, 62, 64, 89-90, 133, 186-193, 204-208, 211-212, 224-226, 231, 240, 251, 312, 318, 344-354, 377, 385-386, 399-400, 425-434, 490, 501, 515-516, 529-540, 564-578, 591-592, 629-630, 653-654, 674-675, 681-682, 698, 712-729, 758-760, 765, 772-774, 790-792, 871, 883, 909-910, 919-941, 965-966, 981-994, 999-1005, 129->133, 167->169, 169->171, 201->204, 223->224, 230->231, 237->239, 239->240, 243->247, 249->251, 309->312, 317->318, 376->377, 381->385, 391->394, 396->399, 424->425, 489->490, 499->501, 514->515, 590->591, 628->629, 652->653, 673->674, 677->687, 680->681, 687->694, 694->698, 755->764, 764->765, 868->871, 880->883, 964->965 src/allmydata/mutable/servermap.py 612 240 186 26 56% 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 74, 130-139, 142, 148, 159-161, 175, 177, 183, 186-199, 206, 213, 217-220, 231, 234-238, 243-252, 255-259, 315, 328-350, 358-363, 370-372, 379, 429, 433, 443-447, 495, 498, 506-508, 514-516, 569-570, 603-611, 623-638, 718-721, 732-741, 792, 796, 803-804, 850-851, 872-874, 911-915, 929-945, 961-975, 982-999, 1003-1013, 1043-1045, 1050-1052, 1060-1064, 1069-1070, 1093-1100, 1106-1186, 1214-1215, 1232-1233, 313->315, 427->429, 432->433, 439->443, 459->461, 493->495, 497->498, 504->506, 509->514, 566->569, 597->603, 687->exit, 702->exit, 710->exit, 717->718, 727->732, 759->exit, 791->792, 795->796, 869->872, 1039->1043, 1047->1050, 1059->1060, 1066->1069, 1092->1093, 1213->1214 -src/allmydata/node.py 388 84 146 34 75% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 738-741, 747, 756, 764, 792-805, 808-809, 814-815, 825-846, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 537->535, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 737->738, 746->747, 763->764, 821->830, 823->821 +src/allmydata/node.py 388 84 146 33 75% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 738-741, 747, 756, 764, 792-805, 808-809, 814-815, 825-846, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 737->738, 746->747, 763->764, 821->830, 823->821 src/allmydata/nodemaker.py 97 23 38 10 70% 49, 61, 66, 70, 81, 94, 107-115, 130-138, 141-150, 57->61, 65->66, 69->70, 79->81, 86->95, 90->94, 104->107, 124->exit, 129->130, 129->133 src/allmydata/scripts/admin.py 51 20 2 1 60% 9-14, 25, 28, 31-37, 40-46, 57, 59, 61-66, 56->57 src/allmydata/scripts/backupdb.py 146 91 14 1 36% 84-91, 94-96, 99, 103, 106, 111-114, 117-119, 122, 125, 128, 176-221, 231-242, 245-263, 266-272, 308-324, 327-333, 336-341, 306->308 @@ -8843,7 +8837,7 @@ src/allmydata/storage/common.py 24 1 4 2 89% 11, 10->11, 36->39 src/allmydata/storage/crawler.py 222 1 64 3 99% 16, 13->16, 96->99, 496->505 src/allmydata/storage/expirer.py 240 1 81 2 99% 9, 7->9, 250->exit -src/allmydata/storage/immutable.py 198 1 48 5 98% 12, 11->12, 142->140, 155->160, 187->197, 273->exit +src/allmydata/storage/immutable.py 198 2 48 6 97% 12, 101, 11->12, 100->101, 142->140, 155->160, 187->197, 273->exit src/allmydata/storage/lease.py 35 1 4 1 95% 12, 11->12 src/allmydata/storage/mutable.py 289 8 90 6 96% 12, 162, 252, 289, 362-367, 11->12, 160->162, 247->252, 307->311, 354->362, 448->exit src/allmydata/storage/server.py 371 9 120 9 96% 13, 92, 222, 243, 329, 349, 375, 422-423, 10->13, 91->92, 221->222, 241->243, 289->300, 317->329, 325->305, 346->349, 374->375 @@ -8905,7 +8899,7 @@ src/allmydata/windows/fixups.py 133 133 54 0 0% 1-237 src/allmydata/windows/registry.py 42 42 12 0 0% 1-77 ------------------------------------------------------------------------------------------------ -TOTAL 27477 11786 8244 602 54% +TOTAL 27477 11800 8244 605 54% 18 files skipped due to complete coverage. + '[' '!' -z 1 ']' ``` Trac: refs #3455 --- src/allmydata/test/test_node.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index bee94861e..ecbf28d80 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -520,7 +520,6 @@ introducer.furl = empty enabled = false [i2p] enabled = false -[node] """ NOLISTEN = """ @@ -566,6 +565,7 @@ class Listeners(unittest.TestCase): create_node_dir(basedir, "testing") with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: f.write(BASE_CONFIG) + f.write("[node]\n") f.write("tub.port = tcp:0\n") f.write("tub.location = AUTO\n") @@ -594,6 +594,7 @@ class Listeners(unittest.TestCase): location = "tcp:localhost:%d,tcp:localhost:%d" % (port1, port2) with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: f.write(BASE_CONFIG) + f.write("[node]\n") f.write("tub.port = %s\n" % port) f.write("tub.location = %s\n" % location) @@ -617,6 +618,7 @@ class Listeners(unittest.TestCase): os.mkdir(os.path.join(basedir, "private")) with open(config_fname, "w") as f: f.write(BASE_CONFIG) + f.write("[node]\n") f.write("tub.port = listen:i2p,listen:tor\n") f.write("tub.location = tcp:example.org:1234\n") config = client.read_config(basedir, "client.port") From baa36157b69ebc3e0375267e873446084db730e6 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Thu, 1 Oct 2020 09:15:18 -0700 Subject: [PATCH 11/24] feat(py3): Update Python internal data model refs As of Python 2.7, some of [the `func_*` and `im_*` attributes also can be accessed through their corresponding `__*__` attributes for Python 3 compatibility](https://docs.python.org/2/reference/datamodel.html?highlight=__self__#the-standard-type-hierarchy). I searched for all such occurrences and this is all that needed changing AFAICT. Converts 9 test errors to errors with new exceptions and improves Python 3 test coverage a smidge: ```diff --- ../../.tox/make-test-py3-all-old.log 2020-10-04 15:16:34.054975263 -0700 +++ ../../.tox/make-test-py3-all-new.log 2020-10-04 15:59:59.355692613 -0700 @@ -2273,9 +2273,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_node.IntroducerNotListening.test_port_none_introducer @@ -5935,9 +5935,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_client.Basic.test_web_apiauthtoken =============================================================================== @@ -7146,9 +7146,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_introducer.Node.test_create @@ -7171,9 +7171,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_introducer.Node.test_furl @@ -7550,9 +7550,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_node.TestCase.test_logdir_is_str =============================================================================== @@ -8448,9 +8448,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.test_system.Connections.test_rref allmydata.test.test_system.SystemTest.test_filesystem @@ -8567,9 +8567,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.web.test_introducer.IntroducerWeb.test_basic_information =============================================================================== @@ -8589,9 +8589,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.web.test_introducer.IntroducerWeb.test_json_front_page =============================================================================== @@ -8611,9 +8611,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.web.test_introducer.IntroducerWeb.test_tahoe_css =============================================================================== @@ -8633,9 +8633,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' allmydata.test.web.test_introducer.IntroducerWeb.test_welcome =============================================================================== @@ -8742,9 +8742,9 @@ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 824, in setup_logging - ob = o.im_self -builtins.AttributeError: 'function' object has no attribute 'im_self' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) +builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + test_exit=1 + ./.tox/py36-coverage/bin/coverage combine + ./.tox/py36-coverage/bin/coverage html @@ -8802,7 +8802,7 @@ src/allmydata/mutable/repairer.py 57 37 18 0 29% 13, 15, 17, 19, 29-34, 65-71, 74-126, 129-131 src/allmydata/mutable/retrieve.py 489 123 120 33 71% 46, 48, 50, 52, 56, 58, 60, 62, 64, 89-90, 133, 186-193, 204-208, 211-212, 224-226, 231, 240, 251, 312, 318, 344-354, 377, 385-386, 399-400, 425-434, 490, 501, 515-516, 529-540, 564-578, 591-592, 629-630, 653-654, 674-675, 681-682, 698, 712-729, 758-760, 765, 772-774, 790-792, 871, 883, 909-910, 919-941, 965-966, 981-994, 999-1005, 129->133, 167->169, 169->171, 201->204, 223->224, 230->231, 237->239, 239->240, 243->247, 249->251, 309->312, 317->318, 376->377, 381->385, 391->394, 396->399, 424->425, 489->490, 499->501, 514->515, 590->591, 628->629, 652->653, 673->674, 677->687, 680->681, 687->694, 694->698, 755->764, 764->765, 868->871, 880->883, 964->965 src/allmydata/mutable/servermap.py 612 240 186 26 56% 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 74, 130-139, 142, 148, 159-161, 175, 177, 183, 186-199, 206, 213, 217-220, 231, 234-238, 243-252, 255-259, 315, 328-350, 358-363, 370-372, 379, 429, 433, 443-447, 495, 498, 506-508, 514-516, 569-570, 603-611, 623-638, 718-721, 732-741, 792, 796, 803-804, 850-851, 872-874, 911-915, 929-945, 961-975, 982-999, 1003-1013, 1043-1045, 1050-1052, 1060-1064, 1069-1070, 1093-1100, 1106-1186, 1214-1215, 1232-1233, 313->315, 427->429, 432->433, 439->443, 459->461, 493->495, 497->498, 504->506, 509->514, 566->569, 597->603, 687->exit, 702->exit, 710->exit, 717->718, 727->732, 759->exit, 791->792, 795->796, 869->872, 1039->1043, 1047->1050, 1059->1060, 1066->1069, 1092->1093, 1213->1214 -src/allmydata/node.py 388 84 146 33 75% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 738-741, 747, 756, 764, 792-805, 808-809, 814-815, 825-846, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 737->738, 746->747, 763->764, 821->830, 823->821 +src/allmydata/node.py 388 82 146 33 76% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 738-741, 747, 756, 764, 792-805, 808-809, 814-815, 827-846, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 737->738, 746->747, 763->764, 821->830, 823->821 src/allmydata/nodemaker.py 97 23 38 10 70% 49, 61, 66, 70, 81, 94, 107-115, 130-138, 141-150, 57->61, 65->66, 69->70, 79->81, 86->95, 90->94, 104->107, 124->exit, 129->130, 129->133 src/allmydata/scripts/admin.py 51 20 2 1 60% 9-14, 25, 28, 31-37, 40-46, 57, 59, 61-66, 56->57 src/allmydata/scripts/backupdb.py 146 91 14 1 36% 84-91, 94-96, 99, 103, 106, 111-114, 117-119, 122, 125, 128, 176-221, 231-242, 245-263, 266-272, 308-324, 327-333, 336-341, 306->308 @@ -8899,7 +8899,7 @@ src/allmydata/windows/fixups.py 133 133 54 0 0% 1-237 src/allmydata/windows/registry.py 42 42 12 0 0% 1-77 ------------------------------------------------------------------------------------------------ -TOTAL 27477 11800 8244 605 54% +TOTAL 27477 11798 8244 605 54% 18 files skipped due to complete coverage. + '[' '!' -z 1 ']' ``` --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 9ba4b0d37..1d04834f4 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -821,7 +821,7 @@ class Node(service.MultiService): for o in twlog.theLogPublisher.observers: # o might be a FileLogObserver's .emit method if type(o) is type(self.setup_logging): # bound method - ob = o.im_self + ob = o.__self__ if isinstance(ob, twlog.FileLogObserver): newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) ob.formatTime = newmeth From a89715ebe80228d17971ec964affc5df8e74f517 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Fri, 2 Oct 2020 10:23:18 -0700 Subject: [PATCH 12/24] feat(py3): feat(py3): Fix use of deprecated type Python 3 did away with the unbound method type entirely, they're just functions under Python 3, and IIUC the unbound type is just an alias for the method type in Python 2. As such, this approach should preserve the behavior under Python 2 and should work under Python 3. With this change, the diff in test output shows one test error converted to a failure, increases coverage in all the modules that have a coverage change, and reveals the next porting bug: ``` ... File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") builtins.TypeError: can't concat str to bytes ``` There are no regressions I see under Python 2, so this seems like the right fix to this particular issue AFAICT. ```diff --- ../../.tox/make-test-py3-all-old.log 2020-10-04 15:59:59.355692613 -0700 +++ ../../.tox/make-test-py3-all-new.log 2020-10-04 16:59:27.870208496 -0700 @@ -1206,7 +1206,7 @@ test_location2 ... [ERROR] test_location_auto_and_explicit ... [ERROR] test_location_not_set ... [ERROR] - test_logdir_is_str ... [ERROR] + test_logdir_is_str ... [FAIL] test_private_config ... [ERROR] test_private_config_missing ... [OK] test_private_config_unreadable ... [ERROR] @@ -2254,7 +2254,7 @@ yield create_introducer(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ self._expectedName, reason.getTraceback()), -twisted.trial.unittest.FailTest: builtins.AttributeError raised instead of ValueError: +twisted.trial.unittest.FailTest: builtins.TypeError raised instead of ValueError: Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks _inlineCallbacks(None, g, status) @@ -2271,11 +2271,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.test_node.IntroducerNotListening.test_port_none_introducer @@ -2307,6 +2305,30 @@ =============================================================================== [FAIL] Traceback (most recent call last): + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 340, in test_logdir_is_str + yield client.create_client(basedir) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks + result = g.send(result) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 298, in create_client_from_config + storage_broker, + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ + node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ + self.setup_logging() + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 842, in setup_logging + foolscap.logging.log.setLogDir(incident_dir.encode(get_filesystem_encoding())) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 336, in call_setLogDir + self.failUnless(isinstance(logdir, str), logdir) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 397, in assertTrue + super(_Assertions, self).assertTrue(condition, msg) + File "/usr/lib/python3.6/unittest/case.py", line 682, in assertTrue + raise self.failureException(msg) +twisted.trial.unittest.FailTest: False is not true : b'/home/rpatterson/src/work/sfu/tahoe-lafs/_trial_temp/test_node/test_logdir_is_str/logs/incidents' + +allmydata.test.test_node.TestCase.test_logdir_is_str +=============================================================================== +[FAIL] +Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks result = g.send(result) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_runner.py", line 192, in test_eliot_destination @@ -5933,11 +5955,9 @@ storage_broker, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.test_client.Basic.test_web_apiauthtoken =============================================================================== @@ -7130,8 +7150,12 @@ =============================================================================== [ERROR] Traceback (most recent call last): -Failure: testtools.testresult.real._StringException: Empty attachments: - twisted-log +Failure: testtools.testresult.real._StringException: twisted-log: {{{ +2020-10-04 23:57:37.636Z [-] Foolscap logging initialized +2020-10-04 23:57:37.636Z [-] Note to developers: twistd.log does not receive very much. +2020-10-04 23:57:37.636Z [-] Use 'flogtool tail -c NODEDIR/private/logport.furl' instead +2020-10-04 23:57:37.636Z [-] and read docs/logging.rst +}}} Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks @@ -7144,19 +7168,21 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +TypeError: can't concat str to bytes allmydata.test.test_introducer.Node.test_create =============================================================================== [ERROR] Traceback (most recent call last): -Failure: testtools.testresult.real._StringException: Empty attachments: - twisted-log +Failure: testtools.testresult.real._StringException: twisted-log: {{{ +2020-10-04 23:57:38.284Z [-] Foolscap logging initialized +2020-10-04 23:57:38.284Z [-] Note to developers: twistd.log does not receive very much. +2020-10-04 23:57:38.284Z [-] Use 'flogtool tail -c NODEDIR/private/logport.furl' instead +2020-10-04 23:57:38.284Z [-] and read docs/logging.rst +}}} Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks @@ -7169,11 +7195,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +TypeError: can't concat str to bytes allmydata.test.test_introducer.Node.test_furl @@ -7540,24 +7564,6 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 340, in test_logdir_is_str - yield client.create_client(basedir) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks - result = g.send(result) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 298, in create_client_from_config - storage_broker, - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ - node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' - -allmydata.test.test_node.TestCase.test_logdir_is_str -=============================================================================== -[ERROR] -Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 259, in test_private_config config = config_from_string(basedir, "", "") File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string @@ -8446,11 +8452,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.test_system.Connections.test_rref allmydata.test.test_system.SystemTest.test_filesystem @@ -8565,11 +8569,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.web.test_introducer.IntroducerWeb.test_basic_information =============================================================================== @@ -8587,11 +8589,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.web.test_introducer.IntroducerWeb.test_json_front_page =============================================================================== @@ -8609,11 +8609,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.web.test_introducer.IntroducerWeb.test_tahoe_css =============================================================================== @@ -8631,11 +8629,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes allmydata.test.web.test_introducer.IntroducerWeb.test_welcome =============================================================================== @@ -8721,7 +8717,7 @@ ------------------------------------------------------------------------------- Ran 1300 tests in ###.###s -FAILED (skips=42, expectedFailures=1, failures=34, errors=532, successes=707) +FAILED (skips=42, expectedFailures=1, failures=35, errors=531, successes=707) Unknown error Traceback (most recent call last): @@ -8740,11 +8736,9 @@ storage_broker, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ - self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) -builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 740, in __init__ + self.config.write_config_file("my_nodeid", b32encode(self.nodeid).lower() + "\n") +builtins.TypeError: can't concat str to bytes + test_exit=1 + ./.tox/py36-coverage/bin/coverage combine + ./.tox/py36-coverage/bin/coverage html @@ -8775,11 +8769,11 @@ src/allmydata/immutable/downloader/__init__.py 7 1 2 1 78% 12, 11->12 src/allmydata/immutable/downloader/common.py 14 1 2 1 88% 12, 11->12 src/allmydata/immutable/downloader/fetcher.py 147 1 56 1 99% 12, 11->12 -src/allmydata/immutable/downloader/finder.py 143 11 40 5 91% 11, 88-89, 108, 115, 170-173, 231-232, 10->11, 87->88, 107->108, 113->115, 165->exit +src/allmydata/immutable/downloader/finder.py 143 9 40 5 92% 11, 88-89, 108, 115, 170-173, 10->11, 87->88, 107->108, 113->115, 165->exit src/allmydata/immutable/downloader/node.py 282 9 66 9 94% 11, 170-172, 224-235, 271, 10->11, 43->exit, 60->68, 165->169, 169->170, 256->exit, 270->271, 517->exit, 537->exit src/allmydata/immutable/downloader/segmentation.py 118 1 22 1 99% 11, 10->11 -src/allmydata/immutable/downloader/share.py 453 23 154 11 94% 11, 210, 239-250, 289-290, 354, 399-400, 430-437, 518, 722-725, 775, 862, 10->11, 209->210, 222->239, 288->289, 352->354, 397->399, 515->518, 660->663, 710->722, 762->exit, 774->775 -src/allmydata/immutable/downloader/status.py 154 12 36 1 93% 11, 65-67, 223, 226, 230, 232, 234, 274, 285, 287, 10->11 +src/allmydata/immutable/downloader/share.py 453 15 154 8 96% 11, 210, 248-250, 354, 430-437, 518, 722-725, 862, 10->11, 209->210, 239->248, 352->354, 515->518, 660->663, 710->722, 762->exit +src/allmydata/immutable/downloader/status.py 154 9 36 1 95% 11, 223, 226, 230, 232, 234, 274, 285, 287, 10->11 src/allmydata/immutable/encode.py 421 13 124 10 95% 114, 197-200, 278-280, 405, 412, 462, 509, 562, 687, 102->104, 113->114, 195->197, 404->405, 411->412, 461->462, 499->509, 505->511, 561->562, 685->687 src/allmydata/immutable/filenode.py 196 48 30 5 72% 77-78, 83, 85, 88, 94-101, 104-124, 127-172, 254, 258, 315, 40->42, 224->227, 227->229, 251->254, 257->258 src/allmydata/immutable/happiness_upload.py 214 1 132 3 99% 15, 13->15, 213->211, 280->279 @@ -8802,7 +8796,7 @@ src/allmydata/mutable/repairer.py 57 37 18 0 29% 13, 15, 17, 19, 29-34, 65-71, 74-126, 129-131 src/allmydata/mutable/retrieve.py 489 123 120 33 71% 46, 48, 50, 52, 56, 58, 60, 62, 64, 89-90, 133, 186-193, 204-208, 211-212, 224-226, 231, 240, 251, 312, 318, 344-354, 377, 385-386, 399-400, 425-434, 490, 501, 515-516, 529-540, 564-578, 591-592, 629-630, 653-654, 674-675, 681-682, 698, 712-729, 758-760, 765, 772-774, 790-792, 871, 883, 909-910, 919-941, 965-966, 981-994, 999-1005, 129->133, 167->169, 169->171, 201->204, 223->224, 230->231, 237->239, 239->240, 243->247, 249->251, 309->312, 317->318, 376->377, 381->385, 391->394, 396->399, 424->425, 489->490, 499->501, 514->515, 590->591, 628->629, 652->653, 673->674, 677->687, 680->681, 687->694, 694->698, 755->764, 764->765, 868->871, 880->883, 964->965 src/allmydata/mutable/servermap.py 612 240 186 26 56% 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 74, 130-139, 142, 148, 159-161, 175, 177, 183, 186-199, 206, 213, 217-220, 231, 234-238, 243-252, 255-259, 315, 328-350, 358-363, 370-372, 379, 429, 433, 443-447, 495, 498, 506-508, 514-516, 569-570, 603-611, 623-638, 718-721, 732-741, 792, 796, 803-804, 850-851, 872-874, 911-915, 929-945, 961-975, 982-999, 1003-1013, 1043-1045, 1050-1052, 1060-1064, 1069-1070, 1093-1100, 1106-1186, 1214-1215, 1232-1233, 313->315, 427->429, 432->433, 439->443, 459->461, 493->495, 497->498, 504->506, 509->514, 566->569, 597->603, 687->exit, 702->exit, 710->exit, 717->718, 727->732, 759->exit, 791->792, 795->796, 869->872, 1039->1043, 1047->1050, 1059->1060, 1066->1069, 1092->1093, 1213->1214 -src/allmydata/node.py 388 82 146 33 76% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 738-741, 747, 756, 764, 792-805, 808-809, 814-815, 827-846, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 737->738, 746->747, 763->764, 821->830, 823->821 +src/allmydata/node.py 388 66 146 33 80% 120, 132, 190, 241, 243-245, 278, 284, 294-295, 303-306, 315, 320, 339, 341, 361, 393-396, 422, 449, 453, 490, 493, 500, 511-512, 548, 566, 574, 581, 583, 590-591, 601, 612, 629-633, 679, 681, 741, 747, 756, 764, 792-805, 808-809, 814-815, 832, 837, 189->190, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 391->393, 421->422, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 547->548, 565->566, 573->574, 580->581, 582->583, 589->590, 600->601, 611->612, 622->629, 673->679, 680->681, 746->747, 763->764, 823->821, 831->832, 835->837 src/allmydata/nodemaker.py 97 23 38 10 70% 49, 61, 66, 70, 81, 94, 107-115, 130-138, 141-150, 57->61, 65->66, 69->70, 79->81, 86->95, 90->94, 104->107, 124->exit, 129->130, 129->133 src/allmydata/scripts/admin.py 51 20 2 1 60% 9-14, 25, 28, 31-37, 40-46, 57, 59, 61-66, 56->57 src/allmydata/scripts/backupdb.py 146 91 14 1 36% 84-91, 94-96, 99, 103, 106, 111-114, 117-119, 122, 125, 128, 176-221, 231-242, 245-263, 266-272, 308-324, 327-333, 336-341, 306->308 @@ -8837,7 +8831,7 @@ src/allmydata/storage/common.py 24 1 4 2 89% 11, 10->11, 36->39 src/allmydata/storage/crawler.py 222 1 64 3 99% 16, 13->16, 96->99, 496->505 src/allmydata/storage/expirer.py 240 1 81 2 99% 9, 7->9, 250->exitp -src/allmydata/storage/immutable.py 198 2 48 6 97% 12, 101, 11->12, 100->101, 142->140, 155->160, 187->197, 273->exit +src/allmydata/storage/immutable.py 198 1 48 5 98% 12, 11->12, 142->140, 155->160, 187->197, 273->exit src/allmydata/storage/lease.py 35 1 4 1 95% 12, 11->12 src/allmydata/storage/mutable.py 289 8 90 6 96% 12, 162, 252, 289, 362-367, 11->12, 160->162, 247->252, 307->311, 354->362, 448->exit src/allmydata/storage/server.py 371 9 120 9 96% 13, 92, 222, 243, 329, 349, 375, 422-423, 10->13, 91->92, 221->222, 241->243, 289->300, 317->329, 325->305, 346->349, 374->375 @@ -8899,7 +8893,7 @@ src/allmydata/windows/fixups.py 133 133 54 0 0% 1-237 src/allmydata/windows/registry.py 42 42 12 0 0% 1-77 ------------------------------------------------------------------------------------------------ -TOTAL 27477 11798 8244 605 54% +TOTAL 27477 11768 8244 601 54% 18 files skipped due to complete coverage. + '[' '!' -z 1 ']' ``` --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 1d04834f4..c7b043a84 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -823,7 +823,7 @@ class Node(service.MultiService): if type(o) is type(self.setup_logging): # bound method ob = o.__self__ if isinstance(ob, twlog.FileLogObserver): - newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) + newmeth = types.MethodType(formatTimeTahoeStyle, ob) ob.formatTime = newmeth # TODO: twisted >2.5.0 offers maxRotatedFiles=50 From da046108e24046915a45674bd4d91b3e3b4fa1f5 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Thu, 1 Oct 2020 09:41:31 -0700 Subject: [PATCH 13/24] feat(py3): Fix section name unicode type clash Before this change, there was an error in the tests in python 3. I couldn't find any clues from the history as to why explicit byte-string literals were used here. This change addresses the error under Python 3 and doesn't cause any regressions under Python 2 in the test suite. This changes two tests from failures to passing under Python 3 and increases coverage a bit: ```diff --- ../../.tox/make-test-py3-all-old.log 2020-10-04 21:42:22.931028265 -0700 +++ ../../.tox/make-test-py3-all-new.log 2020-10-04 21:49:19.164127097 -0700 @@ -313,7 +313,7 @@ ####-##-##T##:##:##-###0 [twisted.scripts._twistd_unix.UnixAppLogger#info] twistd 20.3.0 (/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/bin/python 3.6.12) starting up. ####-##-##T##:##:##-###0 [twisted.scripts._twistd_unix.UnixAppLogger#info] reactor class: mock.mock.MagicMock. ####-##-##T##:##:##-###0 [twisted.scripts._twistd_unix.UnixAppLogger#info] Server Shut Down. - [FAIL] + [OK] allmydata.test cli test_status ... [ERROR] @@ -1191,7 +1191,7 @@ test_disabled_but_helper ... [FAIL] test_disabled_but_storage ... [FAIL] Configuration - test_create_client_invalid_config ... [FAIL] + test_create_client_invalid_config ... [OK] test_read_invalid_config ... [OK] IntroducerNotListening test_port_none_introducer ... [FAIL] @@ -2024,18 +2024,6 @@ =============================================================================== [FAIL] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/mock/mock.py", line 1369, in patched - return func(*newargs, **newkeywargs) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/cli/test_start.py", line 265, in test_run_invalid_config - output, - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 494, in assertIn - % (containee, container)) -twisted.trial.unittest.FailTest: 'invalid section' not in '\nUnknown error\nTraceback (most recent call last):\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/cli/test_start.py", line 232, in cwr\n fn()\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/scripts/run_common.py", line 155, in start\n d = service_factory()\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/scripts/run_common.py", line 136, in \n u"client": lambda: maybeDeferred(namedAny("allmydata.client.create_client"), self.basedir),\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 151, in maybeDeferred\n result = f(*args, **kw)\n--- ---\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 243, in create_client\n config = read_config(basedir, u"client.port")\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 219, in read_config\n _valid_config=_valid_config(),\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 192, in read_config\n configutil.validate_config(config_fname, parser, _valid_config)\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/configutil.py", line 72, in validate_config\n if not valid_config.is_valid_section(section):\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/configutil.py", line 113, in is_valid_section\n self._is_valid_section(section_name)\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/configutil.py", line 141, in \n return lambda *a, **kw: f(*a, **kw) or g(*a, **kw)\n File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 72, in _is_valid_section\n section_name.startswith(b"storageserver.plugins.") or\nbuiltins.TypeError: startswith first arg must be str or a tuple of str, not bytes\n' - -allmydata.test.cli.test_start.RunTests.test_run_invalid_config -=============================================================================== -[FAIL] -Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_asynctest.py", line 75, in _eb raise self.failureException(output) twisted.trial.unittest.FailTest: @@ -2222,34 +2210,6 @@ result = result.throwExceptionIntoGenerator(g) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config - yield client.create_client(self.basedir) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ - self._expectedName, reason.getTraceback()), -twisted.trial.unittest.FailTest: builtins.TypeError raised instead of UnknownConfigError: - Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1529, in _cancellableInlineCallbacks - _inlineCallbacks(None, g, status) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks - result = result.throwExceptionIntoGenerator(g) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator - return g.throw(self.type, self.value, self.tb) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config - yield client.create_client(self.basedir) ---- --- - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 741, in test_create_client_invalid_config - yield client.create_client(self.basedir) -builtins.TypeError: startswith first arg must be str or a tuple of str, not bytes - - -allmydata.test.test_node.Configuration.test_create_client_invalid_config -=============================================================================== -[FAIL] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks - result = result.throwExceptionIntoGenerator(g) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator - return g.throw(self.type, self.value, self.tb) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 704, in test_port_none_introducer yield create_introducer(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 355, in __exit__ @@ -8717,7 +8677,7 @@ ------------------------------------------------------------------------------- Ran 1300 tests in ###.###s -FAILED (skips=42, expectedFailures=1, failures=35, errors=531, successes=707) +FAILED (skips=42, expectedFailures=1, failures=33, errors=531, successes=709) Unknown error Traceback (most recent call last): @@ -8806,7 +8766,7 @@ src/allmydata/scripts/create_node.py 302 98 114 10 66% 224-229, 235, 257-260, 262-265, 268-269, 289-292, 295-298, 329, 339, 347-380, 391-445, 461-477, 223->224, 234->235, 256->257, 261->262, 266->277, 267->268, 288->289, 294->295, 328->329, 338->339 src/allmydata/scripts/debug.py 719 638 202 0 9% 14, 31-32, 35-49, 52-60, 63-142, 146-154, 157-164, 168-217, 220-304, 307-401, 407, 417, 437-465, 468-485, 488-602, 606, 609-611, 637-648, 653-656, 659, 683-689, 692-810, 813-842, 845-848, 851-865, 869, 888, 891-940, 946, 949-950, 957, 960-961, 967-972, 984-985, 999-1000, 1003-1004, 1020-1021, 1025-1031, 1046-1050 src/allmydata/scripts/default_nodedir.py 15 5 6 2 57% 10-14, 9->10, 16->exit -src/allmydata/scripts/run_common.py 135 18 24 6 85% 37, 41-46, 59-60, 149, 158, 192-193, 216-220, 226-227, 55->62, 135->exit, 135->exit, 148->149, 191->192, 225->226 +src/allmydata/scripts/run_common.py 135 17 24 5 86% 37, 41-46, 59-60, 158, 192-193, 216-220, 226-227, 55->62, 135->exit, 135->exit, 191->192, 225->226 src/allmydata/scripts/runner.py 138 49 42 5 61% 84-85, 91, 97-99, 150, 153-160, 174-181, 188-192, 202-232, 237-252, 255, 31->36, 149->150, 151->153, 185->188, 254->255 src/allmydata/scripts/slow_operation.py 69 56 22 0 14% 15-44, 47-52, 55-61, 64-83 src/allmydata/scripts/stats_gatherer.py 44 16 12 3 59% 8, 30, 75-79, 84-93, 7->8, 29->30, 74->75 @@ -8893,7 +8853,7 @@ src/allmydata/windows/fixups.py 133 133 54 0 0% 1-237 src/allmydata/windows/registry.py 42 42 12 0 0% 1-77 ------------------------------------------------------------------------------------------------ -TOTAL 27477 11782 8244 605 54% +TOTAL 27477 11781 8244 604 54% 18 files skipped due to complete coverage. + '[' '!' -z 1 ']' ``` --- src/allmydata/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index dc2e9fcaa..af3a17d48 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -69,8 +69,8 @@ def _is_valid_section(section_name): Currently considers all possible storage server plugin sections valid. """ return ( - section_name.startswith(b"storageserver.plugins.") or - section_name.startswith(b"storageclient.plugins.") + section_name.startswith("storageserver.plugins.") or + section_name.startswith("storageclient.plugins.") ) From f1da68f3408876462de1e359307c1500e03e2180 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Thu, 1 Oct 2020 11:23:52 -0700 Subject: [PATCH 14/24] feat(py3): Fix config from string compatibility I did an audit of the code base and AFAICT the `node.config_from_string(...)` is only used internally. Much of that usage is in tests where most of the usages feed in non-specific, simple `"..."` string literals (IOW, bytes under py2, unicode under py3) while one test module used `b"..."` byte string literals. Given all that it seems to me that the best goal would be to use simple string literals throughout the usage of `node.config_from_string(...)` and have only one special case in that function to handle the difference between versions. I just discovered that running the test with `TEST_SUITE=allmydata` doesn't run the tests in `allmydata.test.test_node` but running them with `TEST_SUITE=allmydata.test.test_node` does run them. I'm trying to figure out why that is, but in the meantime here are the differences in the Python 3 test output when running just the `allmydata.test.test_node` tests. This changes converts 11 tests from errros to success, changes the specific errors for others and improves coverage a bit: ```diff --- ../../.tox/make-test-py3-all-old.log 2020-10-01 11:56:15.428609940 -0700 +++ ../../.tox/make-test-py3-all-new.log 2020-10-01 11:56:55.052792565 -0700 @@ -95,9 +95,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 739, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 831, in setup_logging newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' @@ -158,53 +158,29 @@ (#.### secs) allmydata.test.test_node.TestCase.test_config_required ... [OK] (#.### secs) -allmydata.test.test_node.TestCase.test_location1 ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 112, in test_location1 - tub_location="192.0.2.0:1234") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestCase.test_location1 ... [OK] (#.### secs) allmydata.test.test_node.TestCase.test_location2 ... Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 117, in test_location2 tub_location="192.0.2.0:1234,example.org:8091") File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 65, in testing_tub + cert_filename='DEFAULT_CERTFILE_BLANK' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 669, in create_main_tub + portlocation = _tub_portlocation(config) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 596, in _tub_portlocation + tubport = _convert_tub_port(file_tubport) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 552, in _convert_tub_port + if re.search(r'^\d+$', s): + File "/usr/lib/python3.6/re.py", line 182, in search + return _compile(pattern, flags).search(string) +builtins.TypeError: cannot use a string pattern on a bytes-like object [ERROR] (#.### secs) -allmydata.test.test_node.TestCase.test_location_auto_and_explicit ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 142, in test_location_auto_and_explicit - local_addresses=["127.0.0.1", "192.0.2.0", "example.com:4321"], - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestCase.test_location_auto_and_explicit ... [OK] (#.### secs) -allmydata.test.test_node.TestCase.test_location_not_set ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 128, in test_location_not_set - local_addresses=["127.0.0.1", "192.0.2.0"], - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestCase.test_location_not_set ... [OK] (#.### secs) allmydata.test.test_node.TestCase.test_logdir_is_str ... Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 340, in test_logdir_is_str @@ -215,27 +191,31 @@ storage_broker, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 739, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 831, in setup_logging newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' [ERROR] (#.### secs) allmydata.test.test_node.TestCase.test_private_config ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 259, in test_private_config - config = config_from_string(basedir, "", "") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 261, in test_private_config + self.assertEqual(config.get_private_config("already"), "secret") + File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 434, in assertEqual + super(_Assertions, self).assertEqual(first, second, msg) + File "/usr/lib/python3.6/unittest/case.py", line 829, in assertEqual + assertion_func(first, second, msg=msg) + File "/usr/lib/python3.6/unittest/case.py", line 822, in _baseAssertEqual + raise self.failureException(msg) +twisted.trial.unittest.FailTest: b'secret' != 'secret' +[FAILURE] (#.### secs) allmydata.test.test_node.TestCase.test_private_config_missing ... [OK] (#.### secs) allmydata.test.test_node.TestCase.test_private_config_unreadable ... Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 213, in test_private_config_unreadable config.get_or_create_private_config("foo", "contents") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 369, in get_or_create_private_config + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 374, in get_or_create_private_config fileutil.write(privname, value) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/fileutil.py", line 275, in write f.write(data) @@ -258,77 +238,33 @@ (#.### secs) allmydata.test.test_node.TestCase.test_timestamp ... [OK] (#.### secs) -allmydata.test.test_node.TestCase.test_write_config_unwritable_file ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 288, in test_write_config_unwritable_file - config = config_from_string(basedir, "", "") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestCase.test_write_config_unwritable_file ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_disabled_port_not_tub ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 488, in test_disabled_port_not_tub - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_disabled_port_not_tub ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_disabled_tub_not_port ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 506, in test_disabled_tub_not_port - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_disabled_tub_not_port ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_empty_tub_location ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 470, in test_empty_tub_location - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_empty_tub_location ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_empty_tub_port ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 453, in test_empty_tub_port - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_empty_tub_port ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_parsing_all_disabled ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 439, in test_parsing_all_disabled - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_parsing_all_disabled ... [OK] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_parsing_defaults ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 392, in test_parsing_defaults - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_parsing_defaults ... [OK] (#.### secs) allmydata.test.test_node.TestMissingPorts.test_parsing_location_complex ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 415, in test_parsing_location_complex - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 418, in test_parsing_location_complex + tubport, tublocation = _tub_portlocation(config) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 596, in _tub_portlocation + tubport = _convert_tub_port(file_tubport) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 552, in _convert_tub_port + if re.search(r'^\d+$', s): + File "/usr/lib/python3.6/re.py", line 182, in search + return _compile(pattern, flags).search(string) +builtins.TypeError: cannot use a string pattern on a bytes-like object [ERROR] (#.### secs) -allmydata.test.test_node.TestMissingPorts.test_parsing_tcp ... Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 370, in test_parsing_tcp - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' -[ERROR] +allmydata.test.test_node.TestMissingPorts.test_parsing_tcp ... [OK] (#.### secs) =============================================================================== @@ -415,9 +351,9 @@ tor_provider, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/introducer/server.py", line 87, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 739, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 831, in setup_logging newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' @@ -449,6 +385,20 @@ allmydata.test.test_node.TestCase.test_config_items =============================================================================== +[FAIL] +Traceback (most recent call last): + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 261, in test_private_config + self.assertEqual(config.get_private_config("already"), "secret") + File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/trial/_synctest.py", line 434, in assertEqual + super(_Assertions, self).assertEqual(first, second, msg) + File "/usr/lib/python3.6/unittest/case.py", line 829, in assertEqual + assertion_func(first, second, msg=msg) + File "/usr/lib/python3.6/unittest/case.py", line 822, in _baseAssertEqual + raise self.failureException(msg) +twisted.trial.unittest.FailTest: b'secret' != 'secret' + +allmydata.test.test_node.TestCase.test_private_config +=============================================================================== [ERROR] Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 657, in test_disabled @@ -503,62 +453,26 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 112, in test_location1 - tub_location="192.0.2.0:1234") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestCase.test_location1 -=============================================================================== -[ERROR] -Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 117, in test_location2 tub_location="192.0.2.0:1234,example.org:8091") File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 65, in testing_tub + cert_filename='DEFAULT_CERTFILE_BLANK' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 669, in create_main_tub + portlocation = _tub_portlocation(config) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 596, in _tub_portlocation + tubport = _convert_tub_port(file_tubport) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 552, in _convert_tub_port + if re.search(r'^\d+$', s): + File "/usr/lib/python3.6/re.py", line 182, in search + return _compile(pattern, flags).search(string) +builtins.TypeError: cannot use a string pattern on a bytes-like object allmydata.test.test_node.TestCase.test_location2 =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 142, in test_location_auto_and_explicit - local_addresses=["127.0.0.1", "192.0.2.0", "example.com:4321"], - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestCase.test_location_auto_and_explicit -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 128, in test_location_not_set - local_addresses=["127.0.0.1", "192.0.2.0"], - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 99, in _test_location - tub = testing_tub(config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 53, in testing_tub - config = config_from_string(basedir, 'DEFAULT_PORTNUMFILE_BLANK', config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestCase.test_location_not_set -=============================================================================== -[ERROR] -Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 340, in test_logdir_is_str yield client.create_client(basedir) File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py36-coverage/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks @@ -567,9 +481,9 @@ storage_broker, File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/client.py", line 676, in __init__ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 734, in __init__ + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 739, in __init__ self.setup_logging() - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 826, in setup_logging + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 831, in setup_logging newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) builtins.AttributeError: module 'types' has no attribute 'UnboundMethodType' @@ -577,19 +491,9 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 259, in test_private_config - config = config_from_string(basedir, "", "") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestCase.test_private_config -=============================================================================== -[ERROR] -Traceback (most recent call last): File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 213, in test_private_config_unreadable config.get_or_create_private_config("foo", "contents") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 369, in get_or_create_private_config + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 374, in get_or_create_private_config fileutil.write(privname, value) File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/util/fileutil.py", line 275, in write f.write(data) @@ -607,97 +511,21 @@ =============================================================================== [ERROR] Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 288, in test_write_config_unwritable_file - config = config_from_string(basedir, "", "") - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestCase.test_write_config_unwritable_file -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 488, in test_disabled_port_not_tub - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_disabled_port_not_tub -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 506, in test_disabled_tub_not_port - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_disabled_tub_not_port -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 470, in test_empty_tub_location - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_empty_tub_location -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 453, in test_empty_tub_port - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_empty_tub_port -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 439, in test_parsing_all_disabled - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_parsing_all_disabled -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 392, in test_parsing_defaults - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_parsing_defaults -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 415, in test_parsing_location_complex - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 418, in test_parsing_location_complex + tubport, tublocation = _tub_portlocation(config) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 596, in _tub_portlocation + tubport = _convert_tub_port(file_tubport) + File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 552, in _convert_tub_port + if re.search(r'^\d+$', s): + File "/usr/lib/python3.6/re.py", line 182, in search + return _compile(pattern, flags).search(string) +builtins.TypeError: cannot use a string pattern on a bytes-like object allmydata.test.test_node.TestMissingPorts.test_parsing_location_complex -=============================================================================== -[ERROR] -Traceback (most recent call last): - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/test/test_node.py", line 370, in test_parsing_tcp - config = config_from_string(self.basedir, "portnum", config_data) - File "/home/rpatterson/src/work/sfu/tahoe-lafs/src/allmydata/node.py", line 209, in config_from_string - parser.readfp(BytesIO(config_str)) -builtins.TypeError: a bytes-like object is required, not 'str' - -allmydata.test.test_node.TestMissingPorts.test_parsing_tcp ------------------------------------------------------------------------------- -Ran 34 tests in 2.788s +Ran 34 tests in 2.516s -FAILED (failures=4, errors=21, successes=9) +FAILED (failures=5, errors=9, successes=20) Name Stmts Miss Branch BrPart Cover Missing ------------------------------------------------------------------------------------------------ src/allmydata/__init__.py 16 4 0 0 75% 18-22, 28-32 @@ -751,7 +579,7 @@ src/allmydata/mutable/repairer.py 57 37 18 0 29% 13, 15, 17, 19, 29-34, 65-71, 74-126, 129-131 src/allmydata/mutable/retrieve.py 489 411 120 0 13% 29-43, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 67-69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89-90, 105-160, 164, 167-171, 174-175, 186-193, 201-208, 211-212, 223-227, 230-232, 236-254, 257-275, 278-283, 286-332, 344-354, 362-454, 485-516, 529-540, 564-578, 586-597, 607-633, 643-663, 671-699, 712-729, 738-798, 806-829, 839-889, 897-905, 909-910, 919-941, 950-971, 981-994, 999-1005 src/allmydata/mutable/servermap.py 623 524 198 0 12% 26-38, 41-42, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 70, 72, 74, 76, 78, 80, 82, 116-124, 130-139, 142, 145, 148, 159-161, 165, 170-172, 175, 177, 180-181, 183, 186-199, 202, 206, 213, 217-220, 225-228, 231, 234-238, 243-252, 255-259, 263-265, 269-275, 280-290, 295-305, 311-315, 320-322, 328-350, 358-363, 370-372, 379, 390-450, 454, 457-461, 466-545, 549-557, 560-575, 578-593, 596-613, 623-638, 642-779, 787, 791-799, 803-804, 816-880, 883-904, 910-914, 919-920, 928-944, 960-974, 981-998, 1002-1012, 1020-1183, 1186-1205, 1209-1225, 1228-1229 -src/allmydata/node.py 388 106 146 39 69% 120, 132, 190, 211-213, 241, 243-245, 278, 284, 291-295, 303-306, 315, 320, 339, 341, 361, 368, 370, 377-379, 393-396, 422, 424, 449, 453, 490, 493, 500, 511-512, 547-549, 566, 574, 581, 583, 590-591, 597, 601, 612, 622-634, 679, 681, 736-750, 756, 764, 792-805, 808-809, 814-815, 827-846, 189->190, 204->208, 240->241, 242->243, 277->278, 314->315, 319->320, 338->339, 340->341, 360->361, 365->368, 391->393, 421->422, 423->424, 448->449, 451->453, 489->490, 492->493, 499->500, 510->511, 565->566, 567->570, 573->574, 575->578, 580->581, 582->583, 585->597, 589->590, 600->601, 603->606, 610->616, 611->612, 618->622, 673->679, 680->681, 763->764, 766->768, 821->830, 823->821 +src/allmydata/node.py 391 87 148 30 75% 20, 125, 137, 195, 246, 248-250, 283, 289, 308-311, 320, 325, 344, 346, 366, 373, 375, 382-384, 398-401, 427, 429, 454, 458, 495, 498, 505, 516-517, 606, 617, 634-638, 684, 686, 741-755, 761, 769, 797-810, 813-814, 819-820, 832-851, 19->20, 194->195, 209->213, 245->246, 247->248, 282->283, 319->320, 324->325, 343->344, 345->346, 365->366, 370->373, 396->398, 426->427, 428->429, 453->454, 456->458, 494->495, 497->498, 504->505, 515->516, 605->606, 616->617, 627->634, 678->684, 685->686, 768->769, 771->773, 826->835, 828->826 src/allmydata/nodemaker.py 97 71 38 0 21% 23-33, 36, 38, 41, 44-47, 49, 53-95, 98-115, 118-125, 129-138, 141-150 src/allmydata/scripts/admin.py 51 31 2 0 38% 9-14, 17-21, 25, 28, 31-37, 40-46, 56-57, 59, 61-66, 74-78 src/allmydata/scripts/backupdb.py 146 146 14 0 0% 1-341 @@ -810,7 +638,7 @@ src/allmydata/util/dictutil.py 38 22 12 1 34% 16, 21-24, 27-31, 34-38, 55-56, 59-60, 63-64, 71, 77-78, 12->16 src/allmydata/util/eliotutil.py 115 68 24 0 35% 82-85, 91-94, 104, 117-122, 129-139, 151, 155-159, 163-167, 179-186, 198-199, 202-210, 222-226, 231-247, 250, 266-294, 308-312 src/allmydata/util/encodingutil.py 217 123 80 12 36% 18, 37-38, 41, 43, 52-53, 69, 75-78, 102, 108, 114-122, 130-134, 145-155, 164, 173-175, 178-181, 187, 196-213, 217-231, 237-243, 279-282, 291-296, 314, 320-322, 327, 334-340, 343-355, 358-363, 366-367, 370-373, 379, 395-405, 412-420, 423, 429, 16->18, 36->37, 40->41, 42->43, 66->69, 72->74, 74->75, 278->279, 285->295, 288->291, 299->310, 319->320 -src/allmydata/util/fileutil.py 343 244 120 13 25% 15, 23-25, 47-55, 71-85, 96-97, 100, 103, 106, 109, 115-116, 119-125, 128, 131, 134, 137-138, 142-145, 151-153, 158, 166-176, 179-184, 201-203, 214-237, 241-244, 247-254, 262, 279, 282-290, 293-304, 326, 328, 336-342, 348, 351, 358, 366-376, 382-400, 405, 410-426, 434-462, 486-529, 548-554, 566-568, 573-604, 608-612, 615-627, 633, 636-659, 13->15, 22->23, 200->201, 259->262, 325->326, 327->328, 332->336, 345->351, 347->348, 357->358, 380->382, 404->405, 571->573 +src/allmydata/util/fileutil.py 343 243 120 13 25% 15, 23-25, 47-55, 71-85, 96-97, 100, 103, 106, 109, 115-116, 119-125, 128, 131, 134, 137-138, 142-145, 151-153, 158, 166-176, 179-184, 201-203, 214-237, 241-244, 247-254, 262, 282-290, 293-304, 326, 328, 336-342, 348, 351, 358, 366-376, 382-400, 405, 410-426, 434-462, 486-529, 548-554, 566-568, 573-604, 608-612, 615-627, 633, 636-659, 13->15, 22->23, 200->201, 259->262, 325->326, 327->328, 332->336, 345->351, 347->348, 357->358, 380->382, 404->405, 571->573 src/allmydata/util/gcutil.py 23 3 8 3 81% 20, 51-57, 19->20, 50->51, 64->exit src/allmydata/util/happinessutil.py 77 62 42 1 13% 15, 25-54, 64-69, 82-92, 142-183, 207-223, 235-249, 13->15 src/allmydata/util/hashutil.py 157 76 8 1 50% 14, 40-42, 45-46, 49-56, 60-62, 66-68, 72-76, 118, 122, 126, 130, 134, 138, 142, 146, 150, 154, 158, 162, 166, 174-176, 180-183, 187, 191, 195, 199, 204, 209-210, 214-215, 219, 223-228, 232, 236, 240, 244, 248-250, 254, 258, 262, 266, 270-271, 278, 282, 12->14 @@ -818,7 +646,7 @@ src/allmydata/util/i2p_provider.py 121 73 36 5 35% 44-67, 72-81, 85-135, 151-161, 168, 176-180, 183-184, 187, 193-216, 219, 226, 167->168, 175->176, 182->183, 186->187, 192->193 src/allmydata/util/idlib.py 5 2 0 0 60% 6, 9 src/allmydata/util/iputil.py 172 74 56 12 52% 14, 63-102, 123-140, 151-184, 209, 216, 229, 237-238, 242, 246, 254-257, 271-277, 328-329, 353-354, 13->14, 215->216, 220->242, 226->229, 234->220, 239->234, 245->246, 249->259, 265->261, 291->329, 295->328, 360->exit -src/allmydata/util/log.py 52 27 16 1 38% 13, 38-41, 46-48, 51-61, 67-75, 78, 12->13 +src/allmydata/util/log.py 52 23 16 2 46% 13, 46-48, 51-61, 67-75, 78, 12->13, 39->41 src/allmydata/util/mathutil.py 12 3 2 1 71% 16, 25-26, 15->16 src/allmydata/util/netstring.py 35 24 12 1 26% 13, 31-54, 12->13 src/allmydata/util/observer.py 91 56 20 1 32% 14, 29-32, 36-38, 41, 44, 47, 50-54, 57-60, 63-66, 69-70, 79, 82, 93-97, 103, 106, 109, 112-113, 119-121, 134, 137-139, 142-145, 148-151, 154-157, 13->14 @@ -854,7 +682,7 @@ src/allmydata/windows/fixups.py 133 133 54 0 0% 1-237 src/allmydata/windows/registry.py 42 42 12 0 0% 1-77 ------------------------------------------------------------------------------------------------ -TOTAL 27467 22018 8248 184 17% +TOTAL 27470 21994 8250 176 17% 12 files skipped due to complete coverage. make[#]: Leaving directory '/home/rpatterson/src/work/sfu/tahoe-lafs' ``` --- newsfragments/3455.minor | 2 +- src/allmydata/node.py | 11 ++- src/allmydata/test/common.py | 10 +-- src/allmydata/test/test_client.py | 120 +++++++++++++++--------------- 4 files changed, 74 insertions(+), 69 deletions(-) diff --git a/newsfragments/3455.minor b/newsfragments/3455.minor index fc3ba420a..d7af32b64 100644 --- a/newsfragments/3455.minor +++ b/newsfragments/3455.minor @@ -1 +1 @@ -Cleanup casting to string for better Python 3 compatibility. +Begin porting the `node` module to Python 3. diff --git a/src/allmydata/node.py b/src/allmydata/node.py index c7b043a84..68175aa9e 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -9,11 +9,16 @@ import os.path import re import types import errno -from six.moves import configparser +from io import StringIO import tempfile -from io import BytesIO from base64 import b32decode, b32encode +# BBB: Python 2 compatibility +from six.moves import configparser +from future.utils import PY2 +if PY2: + from io import BytesIO as StringIO # noqa: F811 + from twisted.python import log as twlog from twisted.application import service from twisted.python.failure import Failure @@ -206,7 +211,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None): # load configuration from in-memory string parser = configparser.SafeConfigParser() - parser.readfp(BytesIO(config_str)) + parser.readfp(StringIO(config_str)) fname = "" configutil.validate_config(fname, parser, _valid_config) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 6b0059cb4..d20d45ebf 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -94,9 +94,9 @@ from .common_py3 import LoggingServiceParent, ShouldFailMixin # noqa: F401 TEST_RSA_KEY_SIZE = 522 EMPTY_CLIENT_CONFIG = config_from_string( - b"/dev/null", - b"tub.port", - b"" + "/dev/null", + "tub.port", + "" ) @@ -249,8 +249,8 @@ class UseNode(object): self.config = config_from_string( self.basedir.asTextMode().path, - u"tub.port", -b""" + "tub.port", +""" [node] {node_config} diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index b188008fd..ea6b7bce9 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -252,11 +252,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): is not set. """ config = client.config_from_string( - b"test_storage_default_anonymous_enabled", - b"tub.port", + "test_storage_default_anonymous_enabled", + "tub.port", BASECONFIG + ( - b"[storage]\n" - b"enabled = true\n" + "[storage]\n" + "enabled = true\n" ) ) self.assertTrue(client.anonymous_storage_enabled(config)) @@ -268,11 +268,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ config = client.config_from_string( self.id(), - b"tub.port", + "tub.port", BASECONFIG + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = true\n" + "[storage]\n" + "enabled = true\n" + "anonymous = true\n" ) ) self.assertTrue(client.anonymous_storage_enabled(config)) @@ -284,11 +284,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ config = client.config_from_string( self.id(), - b"tub.port", + "tub.port", BASECONFIG + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = false\n" + "[storage]\n" + "enabled = true\n" + "anonymous = false\n" ) ) self.assertFalse(client.anonymous_storage_enabled(config)) @@ -300,11 +300,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ config = client.config_from_string( self.id(), - b"tub.port", + "tub.port", BASECONFIG + ( - b"[storage]\n" - b"enabled = false\n" - b"anonymous = true\n" + "[storage]\n" + "enabled = false\n" + "anonymous = true\n" ) ) self.assertFalse(client.anonymous_storage_enabled(config)) @@ -680,11 +680,11 @@ class AnonymousStorage(SyncTestCase): os.makedirs(basedir + b"/private") config = client.config_from_string( basedir, - b"tub.port", + "tub.port", BASECONFIG_I % (SOME_FURL,) + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = true\n" + "[storage]\n" + "enabled = true\n" + "anonymous = true\n" ) ) node = yield client.create_client_from_config( @@ -711,11 +711,11 @@ class AnonymousStorage(SyncTestCase): os.makedirs(basedir + b"/private") config = client.config_from_string( basedir, - b"tub.port", + "tub.port", BASECONFIG_I % (SOME_FURL,) + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = false\n" + "[storage]\n" + "enabled = true\n" + "anonymous = false\n" ) ) node = yield client.create_client_from_config( @@ -732,7 +732,7 @@ class AnonymousStorage(SyncTestCase): ]), ) self.expectThat( - config.get_private_config(b"storage.furl", default=None), + config.get_private_config("storage.furl", default=None), Is(None), ) @@ -748,18 +748,18 @@ class AnonymousStorage(SyncTestCase): os.makedirs(basedir + b"/private") enabled_config = client.config_from_string( basedir, - b"tub.port", + "tub.port", BASECONFIG_I % (SOME_FURL,) + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = true\n" + "[storage]\n" + "enabled = true\n" + "anonymous = true\n" ) ) node = yield client.create_client_from_config( enabled_config, _introducer_factory=MemoryIntroducerClient, ) - anonymous_storage_furl = enabled_config.get_private_config(b"storage.furl") + anonymous_storage_furl = enabled_config.get_private_config("storage.furl") def check_furl(): return node.tub.getReferenceForURL(anonymous_storage_furl) # Perform a sanity check that our test code makes sense: is this a @@ -772,11 +772,11 @@ class AnonymousStorage(SyncTestCase): disabled_config = client.config_from_string( basedir, - b"tub.port", + "tub.port", BASECONFIG_I % (SOME_FURL,) + ( - b"[storage]\n" - b"enabled = true\n" - b"anonymous = false\n" + "[storage]\n" + "enabled = true\n" + "anonymous = false\n" ) ) node = yield client.create_client_from_config( @@ -1137,8 +1137,8 @@ class StorageAnnouncementTests(SyncTestCase): create_node_dir(self.basedir, u"") - def get_config(self, storage_enabled, more_storage=b"", more_sections=b""): - return b""" + def get_config(self, storage_enabled, more_storage="", more_sections=""): + return """ [node] tub.location = tcp:192.0.2.0:1234 @@ -1163,7 +1163,7 @@ introducer.furl = pb://abcde@nowhere/fake """ config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config(storage_enabled=False), ) self.assertThat( @@ -1185,7 +1185,7 @@ introducer.furl = pb://abcde@nowhere/fake """ config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config(storage_enabled=True), ) client_deferred = client.create_client_from_config( @@ -1217,13 +1217,13 @@ introducer.furl = pb://abcde@nowhere/fake value = u"thing" config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_storage="plugins=tahoe-lafs-dummy-v1", more_sections=( - b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" - b"some = {}\n".format(value) + "[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + "some = {}\n".format(value) ), ), ) @@ -1258,15 +1258,15 @@ introducer.furl = pb://abcde@nowhere/fake config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2", + more_storage="plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2", more_sections=( - b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" - b"some = thing-1\n" - b"[storageserver.plugins.tahoe-lafs-dummy-v2]\n" - b"some = thing-2\n" + "[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + "some = thing-1\n" + "[storageserver.plugins.tahoe-lafs-dummy-v2]\n" + "some = thing-2\n" ), ), ) @@ -1306,13 +1306,13 @@ introducer.furl = pb://abcde@nowhere/fake config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_storage="plugins=tahoe-lafs-dummy-v1", more_sections=( - b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" - b"some = thing\n" + "[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + "some = thing\n" ), ), ) @@ -1342,10 +1342,10 @@ introducer.furl = pb://abcde@nowhere/fake config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_storage="plugins=tahoe-lafs-dummy-v1", ), ) self.assertThat( @@ -1380,14 +1380,14 @@ introducer.furl = pb://abcde@nowhere/fake config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-v1", + more_storage="plugins=tahoe-lafs-dummy-v1", more_sections=( - b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" + "[storageserver.plugins.tahoe-lafs-dummy-v1]\n" # This will make it explode on instantiation. - b"invalid = configuration\n" + "invalid = configuration\n" ) ), ) @@ -1407,10 +1407,10 @@ introducer.furl = pb://abcde@nowhere/fake """ config = client.config_from_string( self.basedir, - u"tub.port", + "tub.port", self.get_config( storage_enabled=True, - more_storage=b"plugins=tahoe-lafs-dummy-vX", + more_storage="plugins=tahoe-lafs-dummy-vX", ), ) self.assertThat( From 322670febc48b566c11d1c0783168c7fd099cb85 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Sun, 4 Oct 2020 14:10:23 -0700 Subject: [PATCH 15/24] test(coverage): Prevent collection contamination Given that we set `--parallel=True` in `./.coveragerc` then the following could result in collecting data from multiple runs when not intended: $ coverage run ... $ coverage run ... $ coverage combine We may want to include this into the `./tox.ini` configuration too but it will be somewhat annoying to do that win a way that's compatible with Windows for CI. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 861bf67b5..b48e74b0e 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ test: .tox/create-venvs.log ## Run all tests with coverage collection and reporting. test-venv-coverage: # Special handling for reporting coverage even when the test run fails + rm -f ./.coverage.* test_exit= $(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \ $(TEST_SUITE) || test_exit="$$?" From ad4d7f7612089e49104e3400d288ce1f82aa40cd Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 5 Oct 2020 08:34:41 -0400 Subject: [PATCH 16/24] Tweak docstring Since this class is used in multiple test modules now, it makes sense to keep it in common_util instead of test_storage. --- src/allmydata/test/common_util.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 8673e2ece..e3f5cf750 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -129,9 +129,6 @@ class Marker(object): class FakeCanary(object): """For use in storage tests. - - Can be moved back to test_storage.py once enough Python 3 porting has been - done. """ def __init__(self, ignore_disconnectors=False): self.ignore = ignore_disconnectors From b60cd13054b0a0d089e7ce555034f5e7292d0de5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Oct 2020 10:43:48 -0400 Subject: [PATCH 17/24] Fix the integration test failure. --- src/allmydata/_monkeypatch.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/allmydata/_monkeypatch.py b/src/allmydata/_monkeypatch.py index 87475f7de..da37fd979 100644 --- a/src/allmydata/_monkeypatch.py +++ b/src/allmydata/_monkeypatch.py @@ -4,7 +4,6 @@ Monkey-patching of third party libraries. Ported to Python 3. """ - from __future__ import unicode_literals from __future__ import absolute_import from __future__ import division @@ -14,6 +13,8 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from warnings import catch_warnings + def patch(): """Path third-party libraries to make Tahoe-LAFS work.""" @@ -25,7 +26,13 @@ def patch(): # 2. Foolscap is dropped as a dependency. # 3. Tahoe-LAFS drops Python 2 support. - if PY2: + if not PY2: + # Python 3 doesn't need to monkey patch Foolscap + return + + # We need to suppress warnings so as to prevent unexpected output from + # breaking some integration tests. + with catch_warnings(record=True): # Only tested with this version; ensure correctness with new releases, # and then either update the assert or hopefully drop the monkeypatch. from foolscap import __version__ From 1a9c4232aa2f47c6ce93c2393bc4d37be74047c1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Oct 2020 11:14:43 -0400 Subject: [PATCH 18/24] News file --- newsfragments/3462.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3462.minor diff --git a/newsfragments/3462.minor b/newsfragments/3462.minor new file mode 100644 index 000000000..e69de29bb From 3e87ba368ed7549c767a8ae897f4300074136966 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Oct 2020 11:32:19 -0400 Subject: [PATCH 19/24] Port to Python 3. --- src/allmydata/immutable/encode.py | 15 ++++++++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index fb6733cf7..9351df501 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -1,5 +1,18 @@ # -*- test-case-name: allmydata.test.test_encode -*- +""" +Ported to Python 3. +""" + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from zope.interface import implementer from twisted.internet import defer @@ -468,7 +481,7 @@ class Encoder(object): (self, self.segment_size*(segnum+1), self.segment_size*self.num_segments, - 100 * (segnum+1) / self.num_segments, + 100 * (segnum+1) // self.num_segments, ), level=log.OPERATIONAL) elapsed = time.time() - start diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 04b5651db..6be9077b0 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -40,6 +40,7 @@ PORTED_MODULES = [ "allmydata.immutable.downloader.segmentation", "allmydata.immutable.downloader.share", "allmydata.immutable.downloader.status", + "allmydata.immutable.encode", "allmydata.immutable.happiness_upload", "allmydata.immutable.literal", "allmydata.interfaces", From 9dc4f989871d7ee924bda1744aa68ae7c369ff9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Oct 2020 09:29:41 -0400 Subject: [PATCH 20/24] Port to Python 3. --- src/allmydata/immutable/filenode.py | 13 +++++++++++++ src/allmydata/immutable/layout.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 2 ++ 3 files changed, 27 insertions(+) diff --git a/src/allmydata/immutable/filenode.py b/src/allmydata/immutable/filenode.py index 670989c3a..105a8cde3 100644 --- a/src/allmydata/immutable/filenode.py +++ b/src/allmydata/immutable/filenode.py @@ -1,4 +1,17 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from functools import reduce import binascii from time import time as now diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 5d1f691b9..6736e4e06 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import struct from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 04b5651db..f9f09ae9f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -40,7 +40,9 @@ PORTED_MODULES = [ "allmydata.immutable.downloader.segmentation", "allmydata.immutable.downloader.share", "allmydata.immutable.downloader.status", + "allmydata.immutable.filenode", "allmydata.immutable.happiness_upload", + "allmydata.immutable.layout", "allmydata.immutable.literal", "allmydata.interfaces", "allmydata.introducer.interfaces", From a7e5847f52d78d667fb6200f42c87124430f0e65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Oct 2020 09:30:26 -0400 Subject: [PATCH 21/24] News file. --- newsfragments/3463.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3463.minor diff --git a/newsfragments/3463.minor b/newsfragments/3463.minor new file mode 100644 index 000000000..e69de29bb From f2e56887237074880c984bb4917ddcf2f9366bb5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Oct 2020 10:06:02 -0400 Subject: [PATCH 22/24] Port to Python 3. --- src/allmydata/immutable/upload.py | 26 ++++++++++++++++++++------ src/allmydata/util/_python3.py | 1 + 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 5e38ba31a..3fbefd755 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, native_str +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long, unicode import os, time, weakref, itertools @@ -66,7 +78,7 @@ def _serialize_existing_shares(existing_shares): return { server: list(shares) for (server, shares) - in existing_shares.iteritems() + in existing_shares.items() } _EXISTING_SHARES = Field( @@ -79,7 +91,7 @@ def _serialize_happiness_mappings(happiness_mappings): return { sharenum: base32.b2a(serverid) for (sharenum, serverid) - in happiness_mappings.iteritems() + in happiness_mappings.items() } _HAPPINESS_MAPPINGS = Field( @@ -150,7 +162,9 @@ class HelperUploadResults(Copyable, RemoteCopy): # note: don't change this string, it needs to match the value used on the # helper, and it does *not* need to match the fully-qualified # package/module/class name - typeToCopy = "allmydata.upload.UploadResults.tahoe.allmydata.com" + # + # Needs to be native string to make Foolscap happy. + typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com") copytype = typeToCopy # also, think twice about changing the shape of any existing attribute, @@ -283,7 +297,7 @@ class ServerTracker(object): #log.msg("%s._got_reply(%s)" % (self, (alreadygot, buckets))) (alreadygot, buckets) = alreadygot_and_buckets b = {} - for sharenum, rref in buckets.items(): + for sharenum, rref in list(buckets.items()): bp = self.wbp_class(rref, self._server, self.sharesize, self.blocksize, self.num_segments, @@ -780,7 +794,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): shares_to_ask = set() servermap = self._share_placements - for shnum, tracker_id in servermap.items(): + for shnum, tracker_id in list(servermap.items()): if tracker_id == None: continue if tracker.get_serverid() == tracker_id: @@ -1574,7 +1588,7 @@ class AssistedUploader(object): # abbreviated), so if we detect old results, just clobber them. sharemap = upload_results.sharemap - if str in [type(v) for v in sharemap.values()]: + if any(isinstance(v, (bytes, unicode)) for v in sharemap.values()): upload_results.sharemap = None def _build_verifycap(self, helper_upload_results): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2cea7e0aa..066e0ca7c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -45,6 +45,7 @@ PORTED_MODULES = [ "allmydata.immutable.happiness_upload", "allmydata.immutable.layout", "allmydata.immutable.literal", + "allmydata.immutable.upload", "allmydata.interfaces", "allmydata.introducer.interfaces", "allmydata.monitor", From 5b76bf7f4425b280a4a5faf14558846603c92f67 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 7 Oct 2020 10:12:31 -0400 Subject: [PATCH 23/24] Fix trailing whitespace. --- src/allmydata/immutable/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 3fbefd755..e77cbb30b 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -162,7 +162,7 @@ class HelperUploadResults(Copyable, RemoteCopy): # note: don't change this string, it needs to match the value used on the # helper, and it does *not* need to match the fully-qualified # package/module/class name - # + # # Needs to be native string to make Foolscap happy. typeToCopy = native_str("allmydata.upload.UploadResults.tahoe.allmydata.com") copytype = typeToCopy From 95f2d53f92391bb8e427d93b245a64890d1b3092 Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Wed, 7 Oct 2020 12:01:35 -0700 Subject: [PATCH 24/24] chore(refs #3455) Address feedback, BBB comments https://github.com/tahoe-lafs/tahoe-lafs/pull/845#issuecomment-704469561 --- misc/python3/Makefile | 2 +- newsfragments/3464.minor | 1 + src/allmydata/node.py | 2 +- src/allmydata/scripts/common.py | 2 +- src/allmydata/scripts/stats_gatherer.py | 2 +- src/allmydata/scripts/tahoe_check.py | 2 +- src/allmydata/stats.py | 2 +- src/allmydata/test/check_load.py | 2 +- src/allmydata/test/check_memory.py | 2 +- src/allmydata/test/eliotutil.py | 2 +- src/allmydata/test/mutable/test_version.py | 2 +- src/allmydata/test/test_auth.py | 2 +- src/allmydata/test/test_deepcheck.py | 2 +- 13 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3464.minor diff --git a/misc/python3/Makefile b/misc/python3/Makefile index 018d20035..f0ef8b12a 100644 --- a/misc/python3/Makefile +++ b/misc/python3/Makefile @@ -1,4 +1,4 @@ -# BBB: Python 3 porting targets +# Python 3 porting targets # # NOTE: this Makefile requires GNU make diff --git a/newsfragments/3464.minor b/newsfragments/3464.minor new file mode 100644 index 000000000..bc79dee53 --- /dev/null +++ b/newsfragments/3464.minor @@ -0,0 +1 @@ +Cleanup comments that don't match the project convention. diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 68175aa9e..26fc6fc97 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -13,7 +13,7 @@ from io import StringIO import tempfile from base64 import b32decode, b32encode -# BBB: Python 2 compatibility +# Python 2 compatibility from six.moves import configparser from future.utils import PY2 if PY2: diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 13f67abab..b2bb8a1e6 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,7 +4,7 @@ import os, sys, urllib, textwrap import codecs from os.path import join -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/scripts/stats_gatherer.py b/src/allmydata/scripts/stats_gatherer.py index 57136f872..26848a23c 100644 --- a/src/allmydata/scripts/stats_gatherer.py +++ b/src/allmydata/scripts/stats_gatherer.py @@ -2,7 +2,7 @@ from __future__ import print_function import os -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 6531ec285..997ee6e9e 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -3,7 +3,7 @@ from __future__ import print_function import urllib import json -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index e8e3e2965..f669b0861 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -6,7 +6,7 @@ import pprint import time from collections import deque -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 3a8e835de..4058ddf77 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,7 +37,7 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 174b69d4f..41cf6e1d7 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -2,7 +2,7 @@ from __future__ import print_function import os, shutil, sys, urllib, time, stat, urlparse -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 21e109164..9c12629f9 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -2,7 +2,7 @@ Tools aimed at the interaction between tests and Eliot. """ -# BBB: Python 2 compatibility +# Python 2 compatibility # Can't use `builtins.str` because it's not JSON encodable: # `exceptions.TypeError: is not JSON-encodeable` from past.builtins import unicode as str diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 35dea9411..dd871aeb1 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -2,7 +2,7 @@ from __future__ import print_function import os -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py index d6daba24e..5670933df 100644 --- a/src/allmydata/test/test_auth.py +++ b/src/allmydata/test/test_auth.py @@ -1,4 +1,4 @@ -# BBB: Python 2 compatibility +# Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index dcfe1ea35..e3b712715 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -1,6 +1,6 @@ import os, json, urllib -# BBB: Python 2 compatibility +# Python 2 compatibility # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing # a `future.types.newstr.newstr` as a *Python 3* byte string representation under # *Python 2*: