From 996a564c05b5360673c8d14d7697f7e153452443 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 Mar 2021 10:50:44 -0500 Subject: [PATCH 01/22] Private key should be bytes. --- src/allmydata/test/matchers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index eaf7d13a5..2d5e74ffd 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -51,7 +51,7 @@ class MatchesNodePublicKey(object): :return Mismatch: If the keys don't match. """ config = read_config(self.basedir, u"tub.port") - privkey_bytes = config.get_private_config("node.privkey") + privkey_bytes = config.get_private_config("node.privkey").encode("utf-8") private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0] signature = ed25519.sign_data(private_key, b"") other_public_key = ed25519.verifying_key_from_signing_key(other) From 5ec52f9885f946e749496b2bea6158d7c9ef191f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 Mar 2021 14:14:55 -0500 Subject: [PATCH 02/22] More passing tests on Python 3. --- src/allmydata/test/test_client.py | 41 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 63a5ceaaa..07dac634b 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os, sys from functools import ( partial, @@ -186,7 +188,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): basedir, "client.port", ) - abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding()) + abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)) self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0]) @@ -234,7 +236,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) c = yield client.create_client(basedir) - self.failUnless(c.get_long_nodeid().startswith("v0-")) + self.failUnless(c.get_long_nodeid().startswith(b"v0-")) @defer.inlineCallbacks def test_nodekey_no_storage(self): @@ -246,7 +248,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG + "[storage]\n" + "enabled = false\n") c = yield client.create_client(basedir) - self.failUnless(c.get_long_nodeid().startswith("v0-")) + self.failUnless(c.get_long_nodeid().startswith(b"v0-")) def test_storage_anonymous_enabled_by_default(self): """ @@ -431,6 +433,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ generic helper for following storage_dir tests """ + assert isinstance(basedir, unicode) + assert isinstance(storage_path, (unicode, type(None))) + assert isinstance(expected_path, unicode) os.mkdir(basedir) cfg_path = os.path.join(basedir, "tahoe.cfg") fileutil.write( @@ -504,7 +509,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): expected_path = abspath_expanduser_unicode( u"client.Basic.test_absolute_storage_dir_myowndir/" + base ) - config_path = expected_path.encode("utf-8") + config_path = expected_path return self._storage_dir_test( basedir, config_path, @@ -516,32 +521,42 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): def test_permute(self): sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) - for k in ["%d" % i for i in range(5)]: + ks = [b"%d" % i for i in range(5)] + for k in ks: ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) - self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2']) - self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3']) + one = self._permute(sb, b"one") + two = self._permute(sb, b"two") + self.assertEqual(sorted(one), ks) + self.assertEqual(sorted(two), ks) + self.assertNotEqual(one, two) sb.servers.clear() - self.failUnlessReallyEqual(self._permute(sb, "one"), []) + self.failUnlessReallyEqual(self._permute(sb, b"one"), []) def test_permute_with_preferred(self): sb = StorageFarmBroker( True, None, EMPTY_CLIENT_CONFIG, - StorageClientConfig(preferred_peers=['1','4']), + StorageClientConfig(preferred_peers=[b'1',b'4']), ) - for k in ["%d" % i for i in range(5)]: + ks = [b"%d" % i for i in range(5)] + for k in [b"%d" % i for i in range(5)]: ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) - self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2']) - self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3']) + one = self._permute(sb, b"one") + two = self._permute(sb, b"two") + self.assertEqual(sorted(one), ks) + self.assertEqual(sorted(one[:2]), [b"1", b"4"]) + self.assertEqual(sorted(two), ks) + self.assertEqual(sorted(two[:2]), [b"1", b"4"]) + self.assertNotEqual(one, two) sb.servers.clear() - self.failUnlessReallyEqual(self._permute(sb, "one"), []) + self.failUnlessReallyEqual(self._permute(sb, b"one"), []) @defer.inlineCallbacks def test_versions(self): From 2c3353f2fa2ed0863f5edbed495478dd088a1804 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 Mar 2021 14:42:55 -0500 Subject: [PATCH 03/22] More tests passing on Python 3. --- src/allmydata/client.py | 6 +++--- src/allmydata/test/test_client.py | 32 +++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index f5e603490..7f640a8fb 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -364,8 +364,8 @@ class _StoragePlugins(object): """ return set( config.get_config( - "storage", "plugins", b"" - ).decode("ascii").split(u",") + "storage", "plugins", "" + ).split(u",") ) - {u""} @classmethod @@ -870,7 +870,7 @@ class _Client(node.Node, pollmixin.PollMixin): """ Register a storage server. """ - config_key = b"storage-plugin.{}.furl".format( + config_key = "storage-plugin.{}.furl".format( # Oops, why don't I have a better handle on this value? announceable_storage_server.announcement[u"name"], ) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 07dac634b..249be7c38 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -482,7 +482,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): the node's basedir. """ basedir = u"client.Basic.test_relative_storage_dir" - config_path = b"myowndir" + config_path = u"myowndir" expected_path = os.path.join( abspath_expanduser_unicode(basedir), u"myowndir", @@ -572,8 +572,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): c = yield client.create_client(basedir) ss = c.getServiceNamed("storage") verdict = ss.remote_get_version() - self.failUnlessReallyEqual(verdict["application-version"], - str(allmydata.__full_version__)) + self.failUnlessReallyEqual(verdict[b"application-version"], + allmydata.__full_version__.encode("ascii")) self.failIfEqual(str(allmydata.__version__), "unknown") self.failUnless("." in str(allmydata.__full_version__), "non-numeric version in '%s'" % allmydata.__version__) @@ -918,7 +918,7 @@ class Run(unittest.TestCase, testutil.StallMixin): private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child("tahoe.cfg").setContent(BASECONFIG.encode("ascii")) basedir.child(client._Client.EXIT_TRIGGER_FILE).touch() yield client.create_client(basedir.path) @@ -929,7 +929,7 @@ class Run(unittest.TestCase, testutil.StallMixin): private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii")) c1 = yield client.create_client(basedir.path) c1.setServiceParent(self.sparent) @@ -1056,7 +1056,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) c = yield client.create_client(basedir) - n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failUnless(IImmutableFileNode.providedBy(n)) @@ -1074,10 +1074,10 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): # current fix for this (hopefully to be superceded by a better fix # eventually) is to prevent re-use of filenodes, so the NodeMaker is # hereby required *not* to cache and re-use filenodes for CHKs. - other_n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + other_n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") self.failIf(n is other_n, (n, other_n)) - n = c.create_node_from_uri("URI:LIT:n5xgk") + n = c.create_node_from_uri(b"URI:LIT:n5xgk") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failUnless(IImmutableFileNode.providedBy(n)) @@ -1086,7 +1086,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failIf(n.is_mutable()) - n = c.create_node_from_uri("URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1095,7 +1095,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failIf(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1104,7 +1104,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1113,7 +1113,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failIf(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1122,8 +1122,8 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - unknown_rw = "lafs://from_the_future" - unknown_ro = "lafs://readonly_from_the_future" + unknown_rw = b"lafs://from_the_future" + unknown_ro = b"lafs://readonly_from_the_future" n = c.create_node_from_uri(unknown_rw, unknown_ro) self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) @@ -1133,7 +1133,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_unknown()) self.failUnlessReallyEqual(n.get_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw) - self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro) + self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro) # Note: it isn't that we *intend* to deploy non-ASCII caps in # the future, it is that we want to make sure older Tahoe-LAFS @@ -1150,7 +1150,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_unknown()) self.failUnlessReallyEqual(n.get_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw) - self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro) + self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro) From cf418b753a5cf1762d431b317d32e73bb96174af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 Mar 2021 10:51:24 -0500 Subject: [PATCH 04/22] All tests pass on Python 3. --- src/allmydata/storage_client.py | 2 ++ src/allmydata/test/test_client.py | 11 ++++++----- src/allmydata/util/eliotutil.py | 10 +++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 75c554562..b59a7f1bc 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -37,6 +37,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text import re, time, hashlib @@ -198,6 +199,7 @@ class StorageFarmBroker(service.MultiService): # doesn't really matter but it makes the logging behavior more # predictable and easier to test (and at least one test does depend on # this sorted order). + servers = {ensure_text(key): value for (key, value) in servers.items()} for (server_id, server) in sorted(servers.items()): try: storage_server = self._make_storage_server( diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 249be7c38..fdebf4ac2 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,3 +1,4 @@ +from future.utils import PY2 from past.builtins import unicode import os, sys @@ -23,7 +24,6 @@ from hypothesis.strategies import ( ) from eliot.testing import ( - capture_logging, assertHasAction, ) from twisted.trial import unittest @@ -64,6 +64,7 @@ from allmydata.util import ( encodingutil, configutil, ) +from allmydata.util.eliotutil import capture_logging from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode @@ -798,7 +799,7 @@ class StaticServers(Fixture): for (serverid, announcement) in self._server_details }, - })) + }).encode("utf-8")) class StorageClients(SyncTestCase): @@ -847,7 +848,7 @@ class StorageClients(SyncTestCase): succeeded( AfterPreprocessing( get_known_server_details, - Equals([(serverid, announcement)]), + Equals([(serverid.encode("utf-8"), announcement)]), ), ), ) @@ -874,7 +875,7 @@ class StorageClients(SyncTestCase): self.useFixture( StaticServers( self.basedir, - [(serverid, announcement), + [(serverid.encode("ascii"), announcement), # Along with a "bad" server announcement. Order in this list # doesn't matter, yaml serializer and Python dicts are going # to shuffle everything around kind of randomly. @@ -891,7 +892,7 @@ class StorageClients(SyncTestCase): AfterPreprocessing( get_known_server_details, # It should have the good server details. - Equals([(serverid, announcement)]), + Equals([(serverid.encode("utf-8"), announcement)]), ), ), ) diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 9e3cdd3e1..5d144eb1d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -32,7 +32,7 @@ from six import ensure_text from sys import ( stdout, ) -from functools import wraps +from functools import wraps, partial from logging import ( INFO, Handler, @@ -66,6 +66,7 @@ from eliot.twisted import ( DeferredContext, inline_callbacks, ) +from eliot.testing import capture_logging as eliot_capture_logging from twisted.python.usage import ( UsageError, @@ -326,3 +327,10 @@ def log_call_deferred(action_type): return DeferredContext(d).addActionFinish() return logged_f return decorate_log_call_deferred + +# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON +# encoder we want to use when validating messages. +if PY2: + capture_logging = eliot_capture_logging +else: + capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder) From a29c03d34b2c93920f16993abaac1b2b9a93f183 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 4 Mar 2021 11:02:36 -0500 Subject: [PATCH 05/22] Port to Python 3. --- src/allmydata/test/test_client.py | 19 ++++++++++++++----- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index fdebf4ac2..3d48d7eef 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,5 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, sys from functools import ( @@ -189,7 +198,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): basedir, "client.port", ) - abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)) + abs_basedir = fileutil.abspath_expanduser_unicode(str(basedir)) self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0]) @@ -434,9 +443,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ generic helper for following storage_dir tests """ - assert isinstance(basedir, unicode) - assert isinstance(storage_path, (unicode, type(None))) - assert isinstance(expected_path, unicode) + assert isinstance(basedir, str) + assert isinstance(storage_path, (str, type(None))) + assert isinstance(expected_path, str) os.mkdir(basedir) cfg_path = os.path.join(basedir, "tahoe.cfg") fileutil.write( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 39bcd83eb..48df3d5d5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -151,6 +151,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_checker", + "allmydata.test.test_client", "allmydata.test.test_codec", "allmydata.test.test_common_util", "allmydata.test.test_configutil", From f2970f1b161eb43ed20fe619bdfe755942f006b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 10:19:24 -0500 Subject: [PATCH 06/22] Port to Python 3. --- src/allmydata/client.py | 18 ++++++++++++++---- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7f640a8fb..7e93bc0dd 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, stat, time, weakref from base64 import urlsafe_b64encode @@ -460,7 +470,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None): introducers = config.get_introducer_configuration() - for petname, (furl, cache_path) in introducers.items(): + for petname, (furl, cache_path) in list(introducers.items()): ic = _introducer_factory( main_tub, furl.encode("ascii"), @@ -679,7 +689,7 @@ class _Client(node.Node, pollmixin.PollMixin): def init_secrets(self): # configs are always unicode def _unicode_make_secret(): - return unicode(_make_secret(), "ascii") + return str(_make_secret(), "ascii") lease_s = self.config.get_or_create_private_config( "secret", _unicode_make_secret).encode("utf-8") lease_secret = base32.a2b(lease_s) @@ -694,7 +704,7 @@ class _Client(node.Node, pollmixin.PollMixin): def _make_key(): private_key, _ = ed25519.create_signing_keypair() # Config values are always unicode: - return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8") + return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8") private_key_str = self.config.get_or_create_private_config( "node.privkey", _make_key).encode("utf-8") diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 48df3d5d5..82ae41e4e 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -29,6 +29,7 @@ PORTED_MODULES = [ "allmydata._monkeypatch", "allmydata.blacklist", "allmydata.check_results", + "allmydata.client", "allmydata.codec", "allmydata.control", "allmydata.crypto", From c71fa48f3cb852db6de1724b83172912c4e584b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 10:19:48 -0500 Subject: [PATCH 07/22] News file. --- newsfragments/3625.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3625.minor diff --git a/newsfragments/3625.minor b/newsfragments/3625.minor new file mode 100644 index 000000000..e69de29bb From d6fc3c078e421207fef8719c295ec009c441958e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 10:26:38 -0500 Subject: [PATCH 08/22] Port to Python 3. --- src/allmydata/test/matchers.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index 2d5e74ffd..3359a7ed5 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -1,6 +1,16 @@ """ Testtools-style matchers useful to the Tahoe-LAFS test suite. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import attr diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 82ae41e4e..aa7323900 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -87,6 +87,7 @@ PORTED_MODULES = [ "allmydata.storage.server", "allmydata.storage.shares", "allmydata.test.no_network", + "allmydata.test.matchers", "allmydata.test.mutable.util", "allmydata.unknown", "allmydata.uri", From 2b9e1996ec2773ea873a2665110fe565fdb109e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 10:34:56 -0500 Subject: [PATCH 09/22] Fix Python 2 regression. --- src/allmydata/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 7e93bc0dd..3bf976fe5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -8,7 +8,9 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + # Don't use future str to prevent leaking future's newbytes into foolscap, which they break. + from past.builtins import unicode as str import os, stat, time, weakref from base64 import urlsafe_b64encode From 226b6195217a2417b1d167fa3493bb8720095629 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 09:52:48 -0500 Subject: [PATCH 10/22] Don't delete test_client.py. --- nix/tahoe-lafs.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f3ccf950d..c0262768c 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -28,7 +28,6 @@ python.pkgs.buildPythonPackage rec { rm src/allmydata/test/test_i2p_provider.py rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py - rm src/allmydata/test/test_client.py ''; From bc7a9a4b7e53063a92c466650ebce9f84a883df8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 10:38:49 -0500 Subject: [PATCH 11/22] Try to fix Nix a different way. --- nix/tahoe-lafs.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index c0262768c..c831b6b7b 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -24,10 +24,17 @@ python.pkgs.buildPythonPackage rec { # tests with in a module. # Many of these tests don't properly skip when i2p or tor dependencies are - # not supplied (and we are not supplying them). + # not supplied (and we are not supplying them). test_client.py fails because + # version is "unknown" on Nix. + # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter. rm src/allmydata/test/test_i2p_provider.py rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py + rm src/allmydata/test/test_client.py + + # Since we're deleting files, this complains they're missing. For now Nix + # is Python 2-only, anyway, so these tests don't add anything yet. + rm src/allmydata/test/test_python3.py ''; From 6f264a60e341bb9d1dce3945ee59692dc02b2019 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:10:48 -0500 Subject: [PATCH 12/22] A couple tests passing on Python 3. --- src/allmydata/test/test_deepcheck.py | 81 ++++++++++++++-------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index e3b712715..11b6c9d27 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -1,5 +1,3 @@ -import os, json, urllib - # Python 2 compatibility # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing # a `future.types.newstr.newstr` as a *Python 3* byte string representation under @@ -11,7 +9,10 @@ import os, json, urllib # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import native_str +from future.utils import native_str, PY3 + +import os, json +from urllib.parse import quote as url_quote from twisted.trial import unittest from twisted.internet import defer @@ -38,12 +39,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_good(self): self.basedir = "deepcheck/MutableChecker/good" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _created(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) d.addCallback(_created) # now make sure the webapi verifier sees no problems d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", @@ -61,12 +62,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_corrupt(self): self.basedir = "deepcheck/MutableChecker/corrupt" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _stash_and_corrupt(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) self.corrupt_shares_numbered(node.get_uri(), [0], _corrupt_mutable_share_data) d.addCallback(_stash_and_corrupt) @@ -99,12 +100,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_delete_share(self): self.basedir = "deepcheck/MutableChecker/delete_share" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _stash_and_delete(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) self.delete_shares_numbered(node.get_uri(), [0]) d.addCallback(_stash_and_delete) # now make sure the webapi checker notices it @@ -152,7 +153,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, return data def parse_streamed_json(self, s): - for unit in s.split("\n"): + for unit in s.split(b"\n"): if not unit: # stream should end with a newline, so split returns "" continue @@ -165,14 +166,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, @inlineCallbacks def web(self, n, method="GET", **kwargs): # returns (data, url) - url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri()) - + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()])) + url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri()) + + "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()])) data = yield do_http(method, url, browser_like_redirects=True) returnValue((data,url)) @inlineCallbacks def wait_for_operation(self, ophandle): - url = self.client_baseurls[0] + "operations/" + ophandle + url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii") url += "?t=status&output=JSON" while True: body = yield do_http("get", url) @@ -184,7 +185,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, @inlineCallbacks def get_operation_results(self, ophandle, output=None): - url = self.client_baseurls[0] + "operations/" + ophandle + url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii") url += "?t=status" if output: url += "&output=" + output @@ -220,36 +221,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): self.root_uri = n.get_uri() d.addCallback(_created_root) d.addCallback(lambda ign: - c0.create_mutable_file(MutableData("mutable file contents"))) + c0.create_mutable_file(MutableData(b"mutable file contents"))) d.addCallback(lambda n: self.root.set_node(u"mutable", n)) def _created_mutable(n): self.mutable = n self.mutable_uri = n.get_uri() d.addCallback(_created_mutable) - large = upload.Data("Lots of data\n" * 1000, None) + large = upload.Data(b"Lots of data\n" * 1000, None) d.addCallback(lambda ign: self.root.add_file(u"large", large)) def _created_large(n): self.large = n self.large_uri = n.get_uri() d.addCallback(_created_large) - small = upload.Data("Small enough for a LIT", None) + small = upload.Data(b"Small enough for a LIT", None) d.addCallback(lambda ign: self.root.add_file(u"small", small)) def _created_small(n): self.small = n self.small_uri = n.get_uri() d.addCallback(_created_small) - small2 = upload.Data("Small enough for a LIT too", None) + small2 = upload.Data(b"Small enough for a LIT too", None) d.addCallback(lambda ign: self.root.add_file(u"small2", small2)) def _created_small2(n): self.small2 = n self.small2_uri = n.get_uri() d.addCallback(_created_small2) - empty_litdir_uri = "URI:DIR2-LIT:" - tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT + empty_litdir_uri = b"URI:DIR2-LIT:" + tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir")) def _created_empty_lit_dir(n): @@ -397,14 +398,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): mutable = [f for f in files if f["cap"] is not None and f["cap"].startswith("URI:SSK:")][0] - self.failUnlessEqual(mutable["cap"], self.mutable_uri) + self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri) self.failIfEqual(mutable["cap"], mutable["verifycap"]) self.failUnlessEqual(mutable["cap"], mutable["repaircap"]) # for immutable file, verifycap==repaircap!=filecap large = [f for f in files if f["cap"] is not None and f["cap"].startswith("URI:CHK:")][0] - self.failUnlessEqual(large["cap"], self.large_uri) + self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri) self.failIfEqual(large["cap"], large["verifycap"]) self.failUnlessEqual(large["verifycap"], large["repaircap"]) self.check_stats_good(stats) @@ -524,7 +525,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_check_is_healthy(self, data, n, where, incomplete=False): self.failUnlessEqual(data["storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnless("summary" in data, (where, data)) self.failUnlessEqual(data["summary"].lower(), "healthy", "%s: '%s'" % (where, data["summary"])) @@ -562,7 +563,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False): self.failUnlessEqual(data["storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["repair-attempted"], False, where) self.json_check_is_healthy(data["pre-repair-results"], n, where, incomplete) @@ -571,7 +572,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_full_deepcheck_is_healthy(self, data, n, where): self.failUnlessEqual(data["root-storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-healthy"], 3, where) self.failUnlessEqual(data["count-objects-unhealthy"], 0, where) @@ -582,7 +583,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_full_deepcheck_and_repair_is_healthy(self, data, n, where): self.failUnlessEqual(data["root-storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where) @@ -728,6 +729,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_test_cli_good(self, ignored): d = defer.succeed(None) + if PY3: # TODO fixme once Python 3 CLI porting is done + return d d.addCallback(lambda ign: self.do_cli_manifest_stream1()) d.addCallback(lambda ign: self.do_cli_manifest_stream2()) d.addCallback(lambda ign: self.do_cli_manifest_stream3()) @@ -738,7 +741,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): return d def _check_manifest_storage_index(self, out): - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines) @@ -749,7 +752,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 8) caps = {} for l in lines: @@ -794,7 +797,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_verify_cap().to_string() in lines) self.failUnless(self.mutable.get_verify_cap().to_string() in lines) @@ -807,7 +810,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_repair_cap().to_string() in lines) self.failUnless(self.mutable.get_repair_cap().to_string() in lines) @@ -819,7 +822,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): d = self.do_cli("stats", self.root_uri) def _check3(args): (rc, out, err) = args - lines = [l.strip() for l in out.split("\n") if l] + lines = [l.strip() for l in out.split(b"\n") if l] self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines) self.failUnless("count-literal-files: 3" in lines) @@ -905,17 +908,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): d.addCallback(self.create_mangled, "large-unrecoverable") d.addCallback(lambda ignored: c0.create_dirnode()) d.addCallback(self._stash_node, "broken") - large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None) + large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None) d.addCallback(lambda ignored: self.nodes["broken"].add_file(u"large1", large1)) d.addCallback(lambda ignored: self.nodes["broken"].create_subdirectory(u"subdir-good")) - large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None) + large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None) d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2)) d.addCallback(lambda ignored: self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable")) d.addCallback(self._stash_node, "subdir-unrecoverable") - large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None) + large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None) d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3)) d.addCallback(lambda ignored: self._delete_most_shares(self.nodes["broken"])) @@ -928,14 +931,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): def create_mangled(self, ignored, name): nodetype, mangletype = name.split("-", 1) if nodetype == "mutable": - mutable_uploadable = MutableData("mutable file contents") + mutable_uploadable = MutableData(b"mutable file contents") d = self.g.clients[0].create_mutable_file(mutable_uploadable) - d.addCallback(lambda n: self.root.set_node(str(name), n)) + d.addCallback(lambda n: self.root.set_node(str(name), n)) # TODO drop str() once strings are unicode elif nodetype == "large": - large = upload.Data("Lots of data\n" * 1000 + name + "\n", None) + large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None) d = self.root.add_file(str(name), large) elif nodetype == "small": - small = upload.Data("Small enough for a LIT", None) + small = upload.Data(b"Small enough for a LIT", None) d = self.root.add_file(str(name), small) d.addCallback(self._stash_node, name) @@ -1203,7 +1206,7 @@ class Large(DeepCheckBase, unittest.TestCase): kids[u"%03d-small" % i] = (litcap, litcap) return subdir_node.set_children(kids) d.addCallback(_add_children) - up = upload.Data("large enough for CHK" * 100, "") + up = upload.Data(b"large enough for CHK" * 100, "") d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up)) def _start_deepcheck(ignored): From ba6c4adba059113db8e6c69f77db90d229bdaedd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:14:17 -0500 Subject: [PATCH 13/22] All tests pass on Python 3 (albeit skipping some CLI-specific tests). --- src/allmydata/test/common.py | 2 +- src/allmydata/test/test_deepcheck.py | 38 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index f6b6f78d9..533cc4fc6 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1059,7 +1059,7 @@ def _corrupt_mutable_share_data(data, debug=False): assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) data_offset = MutableShareFile.DATA_OFFSET sharetype = data[data_offset:data_offset+1] - assert sharetype == "\x00", "non-SDMF mutable shares not supported" + assert sharetype == b"\x00", "non-SDMF mutable shares not supported" (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, ig_datalen, offsets) = unpack_header(data[data_offset:]) assert version == 0, "this function only handles v0 SDMF files" diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index 11b6c9d27..dbc17c17c 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -50,11 +50,11 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_results(out): - self.failUnless("Healthy : Healthy" in out, out) - self.failUnless("Recoverable Versions: 10*seq1-" in out, out) - self.failIf("Not Healthy!" in out, out) - self.failIf("Unhealthy" in out, out) - self.failIf("Corrupt Shares" in out, out) + self.failUnless(b"Healthy : Healthy" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq1-" in out, out) + self.failIf(b"Not Healthy!" in out, out) + self.failIf(b"Unhealthy" in out, out) + self.failIf(b"Corrupt Shares" in out, out) d.addCallback(_got_results) d.addErrback(self.explain_web_error) return d @@ -75,9 +75,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_results(out): - self.failUnless("Not Healthy!" in out, out) - self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) - self.failUnless("Corrupt Shares:" in out, out) + self.failUnless(b"Not Healthy!" in out, out) + self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) + self.failUnless(b"Corrupt Shares:" in out, out) d.addCallback(_got_results) # now make sure the webapi repairer can fix it @@ -85,13 +85,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): self.GET(self.fileurl+"?t=check&verify=true&repair=true", method="POST")) def _got_repair_results(out): - self.failUnless("
Repair successful
" in out, out) + self.failUnless(b"
Repair successful
" in out, out) d.addCallback(_got_repair_results) d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_postrepair_results(out): - self.failIf("Not Healthy!" in out, out) - self.failUnless("Recoverable Versions: 10*seq" in out, out) + self.failIf(b"Not Healthy!" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq" in out, out) d.addCallback(_got_postrepair_results) d.addErrback(self.explain_web_error) @@ -112,9 +112,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false", method="POST")) def _got_results(out): - self.failUnless("Not Healthy!" in out, out) - self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) - self.failIf("Corrupt Shares" in out, out) + self.failUnless(b"Not Healthy!" in out, out) + self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) + self.failIf(b"Corrupt Shares" in out, out) d.addCallback(_got_results) # now make sure the webapi repairer can fix it @@ -122,13 +122,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): self.GET(self.fileurl+"?t=check&verify=false&repair=true", method="POST")) def _got_repair_results(out): - self.failUnless("Repair successful" in out) + self.failUnless(b"Repair successful" in out) d.addCallback(_got_repair_results) d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false", method="POST")) def _got_postrepair_results(out): - self.failIf("Not Healthy!" in out, out) - self.failUnless("Recoverable Versions: 10*seq" in out) + self.failIf(b"Not Healthy!" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq" in out) d.addCallback(_got_postrepair_results) d.addErrback(self.explain_web_error) @@ -1202,11 +1202,11 @@ class Large(DeepCheckBase, unittest.TestCase): self.subdir_node = subdir_node kids = {} for i in range(1, COUNT): - litcap = LiteralFileURI("%03d-data" % i).to_string() + litcap = LiteralFileURI(b"%03d-data" % i).to_string() kids[u"%03d-small" % i] = (litcap, litcap) return subdir_node.set_children(kids) d.addCallback(_add_children) - up = upload.Data(b"large enough for CHK" * 100, "") + up = upload.Data(b"large enough for CHK" * 100, b"") d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up)) def _start_deepcheck(ignored): From 22fd02f576cb4c31cffb8d272020bb1cfdebac44 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:14:44 -0500 Subject: [PATCH 14/22] News file. --- newsfragments/3628.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3628.minor diff --git a/newsfragments/3628.minor b/newsfragments/3628.minor new file mode 100644 index 000000000..e69de29bb From 4b0b3e9a4d4bbf977fa467576146374fafb6204f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:30:47 -0500 Subject: [PATCH 15/22] Port to Python 3. --- src/allmydata/test/cli/common.py | 2 ++ src/allmydata/test/test_deepcheck.py | 29 ++++++++++++++++++++++------ src/allmydata/util/_python3.py | 5 +++++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index f1c48d1af..3033aed4b 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -52,6 +52,8 @@ class CLITestMixin(ReallyEqualMixin): # Python functions want native strings. So ignore the requirements # for passing arguments to another process and make sure this argument # is a native string. + verb = ensure_str(verb) + args = [ensure_str(arg) for arg in args] client_dir = ensure_str(self.get_clientdir(i=client_num)) nodeargs = [ b"--node-directory", client_dir ] return run_cli(verb, *args, nodeargs=nodeargs, **kwargs) diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index dbc17c17c..74ca27759 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -1,3 +1,11 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + # Python 2 compatibility # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing # a `future.types.newstr.newstr` as a *Python 3* byte string representation under @@ -9,7 +17,10 @@ # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import native_str, PY3 +from future.utils import native_str, PY3, PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + import os, json from urllib.parse import quote as url_quote @@ -30,11 +41,17 @@ from allmydata.uri import LiteralFileURI from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \ ShouldFailMixin -from .common_util import StallMixin, run_cli +from .common_util import StallMixin, run_cli_unicode from .common_web import do_http from allmydata.test.no_network import GridTestMixin from .cli.common import CLITestMixin + +def run_cli(verb, *argv): + """Match usage in existing tests by accept *args.""" + return run_cli_unicode(verb, list(argv)) + + class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_good(self): self.basedir = "deepcheck/MutableChecker/good" @@ -293,7 +310,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): sorted(self.g.get_all_serverids()), where) all_serverids = set() - for (shareid, servers) in cr.get_sharemap().items(): + for (shareid, servers) in list(cr.get_sharemap().items()): all_serverids.update([s.get_serverid() for s in servers]) self.failUnlessEqual(sorted(all_serverids), sorted(self.g.get_all_serverids()), @@ -551,7 +568,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): where) self.failUnless("sharemap" in r, where) all_serverids = set() - for (shareid, serverids_s) in r["sharemap"].items(): + for (shareid, serverids_s) in list(r["sharemap"].items()): all_serverids.update(serverids_s) self.failUnlessEqual(sorted(all_serverids), sorted([idlib.nodeid_b2a(sid) @@ -962,10 +979,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): def _corrupt_some_shares(self, node): for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()): if shnum in (0,1): - yield run_cli("debug", "corrupt-share", native_str(sharefile)) + yield run_cli("debug", "corrupt-share", sharefile) def _delete_most_shares(self, node): - self.delete_shares_numbered(node.get_uri(), range(1,10)) + self.delete_shares_numbered(node.get_uri(), list(range(1,10))) def check_is_healthy(self, cr, where): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index adf831868..be9eb302d 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -161,6 +161,11 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_connection_status", "allmydata.test.test_crawler", "allmydata.test.test_crypto", + + # Only partially ported, CLI-using test code is disabled for now until CLI + # is ported. + "allmydata.test.test_deepcheck", + "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", "allmydata.test.test_dirnode", From 183b80fe2529e3f9e0025c567dfe8b79ee438c2b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:33:52 -0500 Subject: [PATCH 16/22] Fix flake. --- src/allmydata/test/test_deepcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index 74ca27759..baee1acbe 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import native_str, PY3, PY2 +from future.utils import PY3, PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 From 9a7add25999538170ba840d208e0219cd153d53d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:36:11 -0500 Subject: [PATCH 17/22] It was empty. --- src/allmydata/test/test_import.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/allmydata/test/test_import.py diff --git a/src/allmydata/test/test_import.py b/src/allmydata/test/test_import.py deleted file mode 100644 index e69de29bb..000000000 From 719c729d01f78859368e618d31cc08eaf7cc0fac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 9 Mar 2021 11:38:50 -0500 Subject: [PATCH 18/22] Appears to already be ported. --- src/allmydata/util/_python3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index be9eb302d..e5e616674 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -219,4 +219,5 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_util", "allmydata.test.web.test_web", "allmydata.test.web.test_webish", + "allmydata.test.test_windows", ] From 86853dc3d05909b9097fcb697a8f2af8f7853f73 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 10 Mar 2021 13:25:52 -0500 Subject: [PATCH 19/22] Use forked release of txi2p for Python 3 support We've poked https://github.com/str4d/txi2p/issues/10 a few times with requests for a new release, with scant success. So txi2p-tahoe is a thing now. --- newsfragments/3633.installation | 1 + setup.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3633.installation diff --git a/newsfragments/3633.installation b/newsfragments/3633.installation new file mode 100644 index 000000000..8f6d7efdd --- /dev/null +++ b/newsfragments/3633.installation @@ -0,0 +1 @@ +Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. diff --git a/setup.py b/setup.py index ed48bf789..13a16da40 100644 --- a/setup.py +++ b/setup.py @@ -151,8 +151,10 @@ tor_requires = [ ] i2p_requires = [ - # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. - "txi2p; python_version < '3.0'", + # txi2p has Python 3 support, but it's unreleased (see + # https://github.com/str4d/txi2p/issues/10). We could use a fork until + # txi2p's maintainers are back in action. + "txi2p-tahoe", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From 7c03bb6184197d17b49f4fad1db02c3a0154f36e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 10 Mar 2021 14:48:00 -0500 Subject: [PATCH 20/22] Give CI a kick --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 13a16da40..7addd0984 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,7 @@ i2p_requires = [ # txi2p has Python 3 support, but it's unreleased (see # https://github.com/str4d/txi2p/issues/10). We could use a fork until # txi2p's maintainers are back in action. - "txi2p-tahoe", + "txi2p-tahoe >= 0.3.5", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From d24d0519b1cda8b90433327c2952618401453b47 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 12 Mar 2021 15:21:32 -0500 Subject: [PATCH 21/22] Continue using known working txi2p for Python 2 Proceeding with caution here, because the txi2p-tahoe fork is rather hurriedly done, and we are unsure about Tahoe-LAFS' i2p testing story. None of the currently active Tahoe-LAFS contributors use i2p, so we won't know if we are breaking things. --- setup.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 7addd0984..f5ca91b2d 100644 --- a/setup.py +++ b/setup.py @@ -151,10 +151,13 @@ tor_requires = [ ] i2p_requires = [ - # txi2p has Python 3 support, but it's unreleased (see - # https://github.com/str4d/txi2p/issues/10). We could use a fork until - # txi2p's maintainers are back in action. - "txi2p-tahoe >= 0.3.5", + # txi2p has Python 3 support in master branch, but it has not been + # released -- see https://github.com/str4d/txi2p/issues/10. We + # could use a fork for Python 3 until txi2p's maintainers are back + # in action. For Python 2, we could continue using the txi2p + # version about which no one has complained to us so far. + "txi2p; python_version < '3.0'", + "txi2p-tahoe >= 0.3.5; python_version > '3.0'", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': From 6e9a3fa3e9f2cf81e3a28aa895fea784d5074922 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 15 Mar 2021 10:08:18 -0400 Subject: [PATCH 22/22] Add back hard-coded expected results. --- src/allmydata/test/test_client.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 3d48d7eef..957be5197 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -530,6 +530,16 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] def test_permute(self): + """ + Permutations need to be stable across Tahoe releases, which is why we + hardcode a specific expected order. + + This is because the order of these results determines which servers a + client will choose to place shares on and which servers it will consult + (and in what order) when trying to retrieve those shares. If the order + ever changes, all already-placed shares become (at best) harder to find + or (at worst) impossible to find. + """ sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) ks = [b"%d" % i for i in range(5)] for k in ks: @@ -539,6 +549,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): one = self._permute(sb, b"one") two = self._permute(sb, b"two") + self.failUnlessReallyEqual(one, [b'3',b'1',b'0',b'4',b'2']) + self.failUnlessReallyEqual(two, [b'0',b'4',b'2',b'1',b'3']) self.assertEqual(sorted(one), ks) self.assertEqual(sorted(two), ks) self.assertNotEqual(one, two) @@ -546,6 +558,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(self._permute(sb, b"one"), []) def test_permute_with_preferred(self): + """ + Permutations need to be stable across Tahoe releases, which is why we + hardcode a specific expected order. In this case, two values are + preferred and should come first. + """ sb = StorageFarmBroker( True, None, @@ -560,6 +577,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): one = self._permute(sb, b"one") two = self._permute(sb, b"two") + self.failUnlessReallyEqual(b"".join(one), b'14302') + self.failUnlessReallyEqual(b"".join(two), b'41023') self.assertEqual(sorted(one), ks) self.assertEqual(sorted(one[:2]), [b"1", b"4"]) self.assertEqual(sorted(two), ks)