diff --git a/src/allmydata/test/test_import.py b/newsfragments/3625.minor similarity index 100% rename from src/allmydata/test/test_import.py rename to newsfragments/3625.minor diff --git a/newsfragments/3628.minor b/newsfragments/3628.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3633.installation b/newsfragments/3633.installation new file mode 100644 index 000000000..8f6d7efdd --- /dev/null +++ b/newsfragments/3633.installation @@ -0,0 +1 @@ +Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support. diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f3ccf950d..c831b6b7b 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -24,11 +24,17 @@ python.pkgs.buildPythonPackage rec { # tests with in a module. # Many of these tests don't properly skip when i2p or tor dependencies are - # not supplied (and we are not supplying them). + # not supplied (and we are not supplying them). test_client.py fails because + # version is "unknown" on Nix. + # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter. rm src/allmydata/test/test_i2p_provider.py rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py rm src/allmydata/test/test_client.py + + # Since we're deleting files, this complains they're missing. For now Nix + # is Python 2-only, anyway, so these tests don't add anything yet. + rm src/allmydata/test/test_python3.py ''; diff --git a/setup.py b/setup.py index ed48bf789..f5ca91b2d 100644 --- a/setup.py +++ b/setup.py @@ -151,8 +151,13 @@ tor_requires = [ ] i2p_requires = [ - # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10. + # txi2p has Python 3 support in master branch, but it has not been + # released -- see https://github.com/str4d/txi2p/issues/10. We + # could use a fork for Python 3 until txi2p's maintainers are back + # in action. For Python 2, we could continue using the txi2p + # version about which no one has complained to us so far. "txi2p; python_version < '3.0'", + "txi2p-tahoe >= 0.3.5; python_version > '3.0'", ] if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': diff --git a/src/allmydata/client.py b/src/allmydata/client.py index f5e603490..3bf976fe5 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,4 +1,16 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + # Don't use future str to prevent leaking future's newbytes into foolscap, which they break. + from past.builtins import unicode as str import os, stat, time, weakref from base64 import urlsafe_b64encode @@ -364,8 +376,8 @@ class _StoragePlugins(object): """ return set( config.get_config( - "storage", "plugins", b"" - ).decode("ascii").split(u",") + "storage", "plugins", "" + ).split(u",") ) - {u""} @classmethod @@ -460,7 +472,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None): introducers = config.get_introducer_configuration() - for petname, (furl, cache_path) in introducers.items(): + for petname, (furl, cache_path) in list(introducers.items()): ic = _introducer_factory( main_tub, furl.encode("ascii"), @@ -679,7 +691,7 @@ class _Client(node.Node, pollmixin.PollMixin): def init_secrets(self): # configs are always unicode def _unicode_make_secret(): - return unicode(_make_secret(), "ascii") + return str(_make_secret(), "ascii") lease_s = self.config.get_or_create_private_config( "secret", _unicode_make_secret).encode("utf-8") lease_secret = base32.a2b(lease_s) @@ -694,7 +706,7 @@ class _Client(node.Node, pollmixin.PollMixin): def _make_key(): private_key, _ = ed25519.create_signing_keypair() # Config values are always unicode: - return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8") + return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8") private_key_str = self.config.get_or_create_private_config( "node.privkey", _make_key).encode("utf-8") @@ -870,7 +882,7 @@ class _Client(node.Node, pollmixin.PollMixin): """ Register a storage server. """ - config_key = b"storage-plugin.{}.furl".format( + config_key = "storage-plugin.{}.furl".format( # Oops, why don't I have a better handle on this value? announceable_storage_server.announcement[u"name"], ) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0986ded06..b0bcc6835 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -37,6 +37,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text import re, time, hashlib @@ -198,6 +199,7 @@ class StorageFarmBroker(service.MultiService): # doesn't really matter but it makes the logging behavior more # predictable and easier to test (and at least one test does depend on # this sorted order). + servers = {ensure_text(key): value for (key, value) in servers.items()} for (server_id, server) in sorted(servers.items()): try: storage_server = self._make_storage_server( diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index f1c48d1af..3033aed4b 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -52,6 +52,8 @@ class CLITestMixin(ReallyEqualMixin): # Python functions want native strings. So ignore the requirements # for passing arguments to another process and make sure this argument # is a native string. + verb = ensure_str(verb) + args = [ensure_str(arg) for arg in args] client_dir = ensure_str(self.get_clientdir(i=client_num)) nodeargs = [ b"--node-directory", client_dir ] return run_cli(verb, *args, nodeargs=nodeargs, **kwargs) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index f6b6f78d9..533cc4fc6 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1059,7 +1059,7 @@ def _corrupt_mutable_share_data(data, debug=False): assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC) data_offset = MutableShareFile.DATA_OFFSET sharetype = data[data_offset:data_offset+1] - assert sharetype == "\x00", "non-SDMF mutable shares not supported" + assert sharetype == b"\x00", "non-SDMF mutable shares not supported" (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, ig_datalen, offsets) = unpack_header(data[data_offset:]) assert version == 0, "this function only handles v0 SDMF files" diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py index eaf7d13a5..3359a7ed5 100644 --- a/src/allmydata/test/matchers.py +++ b/src/allmydata/test/matchers.py @@ -1,6 +1,16 @@ """ Testtools-style matchers useful to the Tahoe-LAFS test suite. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import attr @@ -51,7 +61,7 @@ class MatchesNodePublicKey(object): :return Mismatch: If the keys don't match. """ config = read_config(self.basedir, u"tub.port") - privkey_bytes = config.get_private_config("node.privkey") + privkey_bytes = config.get_private_config("node.privkey").encode("utf-8") private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0] signature = ed25519.sign_data(private_key, b"") other_public_key = ed25519.verifying_key_from_signing_key(other) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 63a5ceaaa..957be5197 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os, sys from functools import ( partial, @@ -21,7 +33,6 @@ from hypothesis.strategies import ( ) from eliot.testing import ( - capture_logging, assertHasAction, ) from twisted.trial import unittest @@ -62,6 +73,7 @@ from allmydata.util import ( encodingutil, configutil, ) +from allmydata.util.eliotutil import capture_logging from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode @@ -186,7 +198,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): basedir, "client.port", ) - abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding()) + abs_basedir = fileutil.abspath_expanduser_unicode(str(basedir)) self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0]) self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0]) @@ -234,7 +246,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) c = yield client.create_client(basedir) - self.failUnless(c.get_long_nodeid().startswith("v0-")) + self.failUnless(c.get_long_nodeid().startswith(b"v0-")) @defer.inlineCallbacks def test_nodekey_no_storage(self): @@ -246,7 +258,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG + "[storage]\n" + "enabled = false\n") c = yield client.create_client(basedir) - self.failUnless(c.get_long_nodeid().startswith("v0-")) + self.failUnless(c.get_long_nodeid().startswith(b"v0-")) def test_storage_anonymous_enabled_by_default(self): """ @@ -431,6 +443,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): """ generic helper for following storage_dir tests """ + assert isinstance(basedir, str) + assert isinstance(storage_path, (str, type(None))) + assert isinstance(expected_path, str) os.mkdir(basedir) cfg_path = os.path.join(basedir, "tahoe.cfg") fileutil.write( @@ -477,7 +492,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): the node's basedir. """ basedir = u"client.Basic.test_relative_storage_dir" - config_path = b"myowndir" + config_path = u"myowndir" expected_path = os.path.join( abspath_expanduser_unicode(basedir), u"myowndir", @@ -504,7 +519,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): expected_path = abspath_expanduser_unicode( u"client.Basic.test_absolute_storage_dir_myowndir/" + base ) - config_path = expected_path.encode("utf-8") + config_path = expected_path return self._storage_dir_test( basedir, config_path, @@ -515,33 +530,62 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): return [ s.get_longname() for s in sb.get_servers_for_psi(key) ] def test_permute(self): + """ + Permutations need to be stable across Tahoe releases, which is why we + hardcode a specific expected order. + + This is because the order of these results determines which servers a + client will choose to place shares on and which servers it will consult + (and in what order) when trying to retrieve those shares. If the order + ever changes, all already-placed shares become (at best) harder to find + or (at worst) impossible to find. + """ sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG) - for k in ["%d" % i for i in range(5)]: + ks = [b"%d" % i for i in range(5)] + for k in ks: ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) - self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2']) - self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3']) + one = self._permute(sb, b"one") + two = self._permute(sb, b"two") + self.failUnlessReallyEqual(one, [b'3',b'1',b'0',b'4',b'2']) + self.failUnlessReallyEqual(two, [b'0',b'4',b'2',b'1',b'3']) + self.assertEqual(sorted(one), ks) + self.assertEqual(sorted(two), ks) + self.assertNotEqual(one, two) sb.servers.clear() - self.failUnlessReallyEqual(self._permute(sb, "one"), []) + self.failUnlessReallyEqual(self._permute(sb, b"one"), []) def test_permute_with_preferred(self): + """ + Permutations need to be stable across Tahoe releases, which is why we + hardcode a specific expected order. In this case, two values are + preferred and should come first. + """ sb = StorageFarmBroker( True, None, EMPTY_CLIENT_CONFIG, - StorageClientConfig(preferred_peers=['1','4']), + StorageClientConfig(preferred_peers=[b'1',b'4']), ) - for k in ["%d" % i for i in range(5)]: + ks = [b"%d" % i for i in range(5)] + for k in [b"%d" % i for i in range(5)]: ann = {"anonymous-storage-FURL": SOME_FURL, "permutation-seed-base32": base32.b2a(k) } sb.test_add_rref(k, "rref", ann) - self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2']) - self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3']) + one = self._permute(sb, b"one") + two = self._permute(sb, b"two") + self.failUnlessReallyEqual(b"".join(one), b'14302') + self.failUnlessReallyEqual(b"".join(two), b'41023') + self.assertEqual(sorted(one), ks) + self.assertEqual(sorted(one[:2]), [b"1", b"4"]) + self.assertEqual(sorted(two), ks) + self.assertEqual(sorted(two[:2]), [b"1", b"4"]) + self.assertNotEqual(one, two) sb.servers.clear() - self.failUnlessReallyEqual(self._permute(sb, "one"), []) + self.failUnlessReallyEqual(self._permute(sb, b"one"), []) @defer.inlineCallbacks def test_versions(self): @@ -557,8 +601,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): c = yield client.create_client(basedir) ss = c.getServiceNamed("storage") verdict = ss.remote_get_version() - self.failUnlessReallyEqual(verdict["application-version"], - str(allmydata.__full_version__)) + self.failUnlessReallyEqual(verdict[b"application-version"], + allmydata.__full_version__.encode("ascii")) self.failIfEqual(str(allmydata.__version__), "unknown") self.failUnless("." in str(allmydata.__full_version__), "non-numeric version in '%s'" % allmydata.__version__) @@ -783,7 +827,7 @@ class StaticServers(Fixture): for (serverid, announcement) in self._server_details }, - })) + }).encode("utf-8")) class StorageClients(SyncTestCase): @@ -832,7 +876,7 @@ class StorageClients(SyncTestCase): succeeded( AfterPreprocessing( get_known_server_details, - Equals([(serverid, announcement)]), + Equals([(serverid.encode("utf-8"), announcement)]), ), ), ) @@ -859,7 +903,7 @@ class StorageClients(SyncTestCase): self.useFixture( StaticServers( self.basedir, - [(serverid, announcement), + [(serverid.encode("ascii"), announcement), # Along with a "bad" server announcement. Order in this list # doesn't matter, yaml serializer and Python dicts are going # to shuffle everything around kind of randomly. @@ -876,7 +920,7 @@ class StorageClients(SyncTestCase): AfterPreprocessing( get_known_server_details, # It should have the good server details. - Equals([(serverid, announcement)]), + Equals([(serverid.encode("utf-8"), announcement)]), ), ), ) @@ -903,7 +947,7 @@ class Run(unittest.TestCase, testutil.StallMixin): private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child("tahoe.cfg").setContent(BASECONFIG.encode("ascii")) basedir.child(client._Client.EXIT_TRIGGER_FILE).touch() yield client.create_client(basedir.path) @@ -914,7 +958,7 @@ class Run(unittest.TestCase, testutil.StallMixin): private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii")) c1 = yield client.create_client(basedir.path) c1.setServiceParent(self.sparent) @@ -1041,7 +1085,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) c = yield client.create_client(basedir) - n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failUnless(IImmutableFileNode.providedBy(n)) @@ -1059,10 +1103,10 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): # current fix for this (hopefully to be superceded by a better fix # eventually) is to prevent re-use of filenodes, so the NodeMaker is # hereby required *not* to cache and re-use filenodes for CHKs. - other_n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") + other_n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277") self.failIf(n is other_n, (n, other_n)) - n = c.create_node_from_uri("URI:LIT:n5xgk") + n = c.create_node_from_uri(b"URI:LIT:n5xgk") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failUnless(IImmutableFileNode.providedBy(n)) @@ -1071,7 +1115,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failIf(n.is_mutable()) - n = c.create_node_from_uri("URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1080,7 +1124,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failIf(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failUnless(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1089,7 +1133,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1098,7 +1142,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failIf(n.is_readonly()) self.failUnless(n.is_mutable()) - n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") + n = c.create_node_from_uri(b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq") self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) self.failIf(IImmutableFileNode.providedBy(n)) @@ -1107,8 +1151,8 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_readonly()) self.failUnless(n.is_mutable()) - unknown_rw = "lafs://from_the_future" - unknown_ro = "lafs://readonly_from_the_future" + unknown_rw = b"lafs://from_the_future" + unknown_ro = b"lafs://readonly_from_the_future" n = c.create_node_from_uri(unknown_rw, unknown_ro) self.failUnless(IFilesystemNode.providedBy(n)) self.failIf(IFileNode.providedBy(n)) @@ -1118,7 +1162,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_unknown()) self.failUnlessReallyEqual(n.get_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw) - self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro) + self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro) # Note: it isn't that we *intend* to deploy non-ASCII caps in # the future, it is that we want to make sure older Tahoe-LAFS @@ -1135,7 +1179,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase): self.failUnless(n.is_unknown()) self.failUnlessReallyEqual(n.get_uri(), unknown_rw) self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw) - self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro) + self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro) diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py index e3b712715..baee1acbe 100644 --- a/src/allmydata/test/test_deepcheck.py +++ b/src/allmydata/test/test_deepcheck.py @@ -1,4 +1,10 @@ -import os, json, urllib +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # Python 2 compatibility # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing @@ -11,7 +17,13 @@ import os, json, urllib # (Pdb) pp data # '334:12:b\'mutable-good\',90:URI:SSK-RO:... from past.builtins import unicode as str -from future.utils import native_str +from future.utils import PY3, PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + + +import os, json +from urllib.parse import quote as url_quote from twisted.trial import unittest from twisted.internet import defer @@ -29,31 +41,37 @@ from allmydata.uri import LiteralFileURI from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \ ShouldFailMixin -from .common_util import StallMixin, run_cli +from .common_util import StallMixin, run_cli_unicode from .common_web import do_http from allmydata.test.no_network import GridTestMixin from .cli.common import CLITestMixin + +def run_cli(verb, *argv): + """Match usage in existing tests by accept *args.""" + return run_cli_unicode(verb, list(argv)) + + class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_good(self): self.basedir = "deepcheck/MutableChecker/good" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _created(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) d.addCallback(_created) # now make sure the webapi verifier sees no problems d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_results(out): - self.failUnless("Healthy : Healthy" in out, out) - self.failUnless("Recoverable Versions: 10*seq1-" in out, out) - self.failIf("Not Healthy!" in out, out) - self.failIf("Unhealthy" in out, out) - self.failIf("Corrupt Shares" in out, out) + self.failUnless(b"Healthy : Healthy" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq1-" in out, out) + self.failIf(b"Not Healthy!" in out, out) + self.failIf(b"Unhealthy" in out, out) + self.failIf(b"Corrupt Shares" in out, out) d.addCallback(_got_results) d.addErrback(self.explain_web_error) return d @@ -61,12 +79,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_corrupt(self): self.basedir = "deepcheck/MutableChecker/corrupt" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _stash_and_corrupt(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) self.corrupt_shares_numbered(node.get_uri(), [0], _corrupt_mutable_share_data) d.addCallback(_stash_and_corrupt) @@ -74,9 +92,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_results(out): - self.failUnless("Not Healthy!" in out, out) - self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) - self.failUnless("Corrupt Shares:" in out, out) + self.failUnless(b"Not Healthy!" in out, out) + self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) + self.failUnless(b"Corrupt Shares:" in out, out) d.addCallback(_got_results) # now make sure the webapi repairer can fix it @@ -84,13 +102,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): self.GET(self.fileurl+"?t=check&verify=true&repair=true", method="POST")) def _got_repair_results(out): - self.failUnless("
Repair successful
" in out, out) + self.failUnless(b"
Repair successful
" in out, out) d.addCallback(_got_repair_results) d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", method="POST")) def _got_postrepair_results(out): - self.failIf("Not Healthy!" in out, out) - self.failUnless("Recoverable Versions: 10*seq" in out, out) + self.failIf(b"Not Healthy!" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq" in out, out) d.addCallback(_got_postrepair_results) d.addErrback(self.explain_web_error) @@ -99,21 +117,21 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): def test_delete_share(self): self.basedir = "deepcheck/MutableChecker/delete_share" self.set_up_grid() - CONTENTS = "a little bit of data" + CONTENTS = b"a little bit of data" CONTENTS_uploadable = MutableData(CONTENTS) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) def _stash_and_delete(node): self.node = node - self.fileurl = "uri/" + urllib.quote(node.get_uri()) + self.fileurl = "uri/" + url_quote(node.get_uri()) self.delete_shares_numbered(node.get_uri(), [0]) d.addCallback(_stash_and_delete) # now make sure the webapi checker notices it d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false", method="POST")) def _got_results(out): - self.failUnless("Not Healthy!" in out, out) - self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) - self.failIf("Corrupt Shares" in out, out) + self.failUnless(b"Not Healthy!" in out, out) + self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out) + self.failIf(b"Corrupt Shares" in out, out) d.addCallback(_got_results) # now make sure the webapi repairer can fix it @@ -121,13 +139,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin): self.GET(self.fileurl+"?t=check&verify=false&repair=true", method="POST")) def _got_repair_results(out): - self.failUnless("Repair successful" in out) + self.failUnless(b"Repair successful" in out) d.addCallback(_got_repair_results) d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false", method="POST")) def _got_postrepair_results(out): - self.failIf("Not Healthy!" in out, out) - self.failUnless("Recoverable Versions: 10*seq" in out) + self.failIf(b"Not Healthy!" in out, out) + self.failUnless(b"Recoverable Versions: 10*seq" in out) d.addCallback(_got_postrepair_results) d.addErrback(self.explain_web_error) @@ -152,7 +170,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, return data def parse_streamed_json(self, s): - for unit in s.split("\n"): + for unit in s.split(b"\n"): if not unit: # stream should end with a newline, so split returns "" continue @@ -165,14 +183,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, @inlineCallbacks def web(self, n, method="GET", **kwargs): # returns (data, url) - url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri()) - + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()])) + url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri()) + + "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()])) data = yield do_http(method, url, browser_like_redirects=True) returnValue((data,url)) @inlineCallbacks def wait_for_operation(self, ophandle): - url = self.client_baseurls[0] + "operations/" + ophandle + url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii") url += "?t=status&output=JSON" while True: body = yield do_http("get", url) @@ -184,7 +202,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin, @inlineCallbacks def get_operation_results(self, ophandle, output=None): - url = self.client_baseurls[0] + "operations/" + ophandle + url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii") url += "?t=status" if output: url += "&output=" + output @@ -220,36 +238,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): self.root_uri = n.get_uri() d.addCallback(_created_root) d.addCallback(lambda ign: - c0.create_mutable_file(MutableData("mutable file contents"))) + c0.create_mutable_file(MutableData(b"mutable file contents"))) d.addCallback(lambda n: self.root.set_node(u"mutable", n)) def _created_mutable(n): self.mutable = n self.mutable_uri = n.get_uri() d.addCallback(_created_mutable) - large = upload.Data("Lots of data\n" * 1000, None) + large = upload.Data(b"Lots of data\n" * 1000, None) d.addCallback(lambda ign: self.root.add_file(u"large", large)) def _created_large(n): self.large = n self.large_uri = n.get_uri() d.addCallback(_created_large) - small = upload.Data("Small enough for a LIT", None) + small = upload.Data(b"Small enough for a LIT", None) d.addCallback(lambda ign: self.root.add_file(u"small", small)) def _created_small(n): self.small = n self.small_uri = n.get_uri() d.addCallback(_created_small) - small2 = upload.Data("Small enough for a LIT too", None) + small2 = upload.Data(b"Small enough for a LIT too", None) d.addCallback(lambda ign: self.root.add_file(u"small2", small2)) def _created_small2(n): self.small2 = n self.small2_uri = n.get_uri() d.addCallback(_created_small2) - empty_litdir_uri = "URI:DIR2-LIT:" - tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT + empty_litdir_uri = b"URI:DIR2-LIT:" + tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir")) def _created_empty_lit_dir(n): @@ -292,7 +310,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): sorted(self.g.get_all_serverids()), where) all_serverids = set() - for (shareid, servers) in cr.get_sharemap().items(): + for (shareid, servers) in list(cr.get_sharemap().items()): all_serverids.update([s.get_serverid() for s in servers]) self.failUnlessEqual(sorted(all_serverids), sorted(self.g.get_all_serverids()), @@ -397,14 +415,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): mutable = [f for f in files if f["cap"] is not None and f["cap"].startswith("URI:SSK:")][0] - self.failUnlessEqual(mutable["cap"], self.mutable_uri) + self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri) self.failIfEqual(mutable["cap"], mutable["verifycap"]) self.failUnlessEqual(mutable["cap"], mutable["repaircap"]) # for immutable file, verifycap==repaircap!=filecap large = [f for f in files if f["cap"] is not None and f["cap"].startswith("URI:CHK:")][0] - self.failUnlessEqual(large["cap"], self.large_uri) + self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri) self.failIfEqual(large["cap"], large["verifycap"]) self.failUnlessEqual(large["verifycap"], large["repaircap"]) self.check_stats_good(stats) @@ -524,7 +542,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_check_is_healthy(self, data, n, where, incomplete=False): self.failUnlessEqual(data["storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnless("summary" in data, (where, data)) self.failUnlessEqual(data["summary"].lower(), "healthy", "%s: '%s'" % (where, data["summary"])) @@ -550,7 +568,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): where) self.failUnless("sharemap" in r, where) all_serverids = set() - for (shareid, serverids_s) in r["sharemap"].items(): + for (shareid, serverids_s) in list(r["sharemap"].items()): all_serverids.update(serverids_s) self.failUnlessEqual(sorted(all_serverids), sorted([idlib.nodeid_b2a(sid) @@ -562,7 +580,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False): self.failUnlessEqual(data["storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["repair-attempted"], False, where) self.json_check_is_healthy(data["pre-repair-results"], n, where, incomplete) @@ -571,7 +589,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_full_deepcheck_is_healthy(self, data, n, where): self.failUnlessEqual(data["root-storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-healthy"], 3, where) self.failUnlessEqual(data["count-objects-unhealthy"], 0, where) @@ -582,7 +600,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def json_full_deepcheck_and_repair_is_healthy(self, data, n, where): self.failUnlessEqual(data["root-storage-index"], - base32.b2a(n.get_storage_index()), where) + str(base32.b2a(n.get_storage_index()), "ascii"), where) self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where) @@ -728,6 +746,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def do_test_cli_good(self, ignored): d = defer.succeed(None) + if PY3: # TODO fixme once Python 3 CLI porting is done + return d d.addCallback(lambda ign: self.do_cli_manifest_stream1()) d.addCallback(lambda ign: self.do_cli_manifest_stream2()) d.addCallback(lambda ign: self.do_cli_manifest_stream3()) @@ -738,7 +758,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): return d def _check_manifest_storage_index(self, out): - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines) @@ -749,7 +769,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 8) caps = {} for l in lines: @@ -794,7 +814,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_verify_cap().to_string() in lines) self.failUnless(self.mutable.get_verify_cap().to_string() in lines) @@ -807,7 +827,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessEqual(err, "") - lines = [l for l in out.split("\n") if l] + lines = [l for l in out.split(b"\n") if l] self.failUnlessEqual(len(lines), 3) self.failUnless(self.root.get_repair_cap().to_string() in lines) self.failUnless(self.mutable.get_repair_cap().to_string() in lines) @@ -819,7 +839,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase): d = self.do_cli("stats", self.root_uri) def _check3(args): (rc, out, err) = args - lines = [l.strip() for l in out.split("\n") if l] + lines = [l.strip() for l in out.split(b"\n") if l] self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines) self.failUnless("count-literal-files: 3" in lines) @@ -905,17 +925,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): d.addCallback(self.create_mangled, "large-unrecoverable") d.addCallback(lambda ignored: c0.create_dirnode()) d.addCallback(self._stash_node, "broken") - large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None) + large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None) d.addCallback(lambda ignored: self.nodes["broken"].add_file(u"large1", large1)) d.addCallback(lambda ignored: self.nodes["broken"].create_subdirectory(u"subdir-good")) - large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None) + large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None) d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2)) d.addCallback(lambda ignored: self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable")) d.addCallback(self._stash_node, "subdir-unrecoverable") - large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None) + large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None) d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3)) d.addCallback(lambda ignored: self._delete_most_shares(self.nodes["broken"])) @@ -928,14 +948,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): def create_mangled(self, ignored, name): nodetype, mangletype = name.split("-", 1) if nodetype == "mutable": - mutable_uploadable = MutableData("mutable file contents") + mutable_uploadable = MutableData(b"mutable file contents") d = self.g.clients[0].create_mutable_file(mutable_uploadable) - d.addCallback(lambda n: self.root.set_node(str(name), n)) + d.addCallback(lambda n: self.root.set_node(str(name), n)) # TODO drop str() once strings are unicode elif nodetype == "large": - large = upload.Data("Lots of data\n" * 1000 + name + "\n", None) + large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None) d = self.root.add_file(str(name), large) elif nodetype == "small": - small = upload.Data("Small enough for a LIT", None) + small = upload.Data(b"Small enough for a LIT", None) d = self.root.add_file(str(name), small) d.addCallback(self._stash_node, name) @@ -959,10 +979,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase): def _corrupt_some_shares(self, node): for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()): if shnum in (0,1): - yield run_cli("debug", "corrupt-share", native_str(sharefile)) + yield run_cli("debug", "corrupt-share", sharefile) def _delete_most_shares(self, node): - self.delete_shares_numbered(node.get_uri(), range(1,10)) + self.delete_shares_numbered(node.get_uri(), list(range(1,10))) def check_is_healthy(self, cr, where): @@ -1199,11 +1219,11 @@ class Large(DeepCheckBase, unittest.TestCase): self.subdir_node = subdir_node kids = {} for i in range(1, COUNT): - litcap = LiteralFileURI("%03d-data" % i).to_string() + litcap = LiteralFileURI(b"%03d-data" % i).to_string() kids[u"%03d-small" % i] = (litcap, litcap) return subdir_node.set_children(kids) d.addCallback(_add_children) - up = upload.Data("large enough for CHK" * 100, "") + up = upload.Data(b"large enough for CHK" * 100, b"") d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up)) def _start_deepcheck(ignored): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 51dd8111d..dae20e988 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -29,6 +29,7 @@ PORTED_MODULES = [ "allmydata._monkeypatch", "allmydata.blacklist", "allmydata.check_results", + "allmydata.client", "allmydata.codec", "allmydata.control", "allmydata.crypto", @@ -88,6 +89,7 @@ PORTED_MODULES = [ "allmydata.storage.server", "allmydata.storage.shares", "allmydata.test.no_network", + "allmydata.test.matchers", "allmydata.test.mutable.util", "allmydata.unknown", "allmydata.uri", @@ -159,12 +161,18 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_base32", "allmydata.test.test_base62", "allmydata.test.test_checker", + "allmydata.test.test_client", "allmydata.test.test_codec", "allmydata.test.test_common_util", "allmydata.test.test_configutil", "allmydata.test.test_connection_status", "allmydata.test.test_crawler", "allmydata.test.test_crypto", + + # Only partially ported, CLI-using test code is disabled for now until CLI + # is ported. + "allmydata.test.test_deepcheck", + "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", "allmydata.test.test_dirnode", @@ -219,4 +227,5 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_util", "allmydata.test.web.test_web", "allmydata.test.web.test_webish", + "allmydata.test.test_windows", ] diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py index 9e3cdd3e1..5d144eb1d 100644 --- a/src/allmydata/util/eliotutil.py +++ b/src/allmydata/util/eliotutil.py @@ -32,7 +32,7 @@ from six import ensure_text from sys import ( stdout, ) -from functools import wraps +from functools import wraps, partial from logging import ( INFO, Handler, @@ -66,6 +66,7 @@ from eliot.twisted import ( DeferredContext, inline_callbacks, ) +from eliot.testing import capture_logging as eliot_capture_logging from twisted.python.usage import ( UsageError, @@ -326,3 +327,10 @@ def log_call_deferred(action_type): return DeferredContext(d).addActionFinish() return logged_f return decorate_log_call_deferred + +# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON +# encoder we want to use when validating messages. +if PY2: + capture_logging = eliot_capture_logging +else: + capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder)