diff --git a/newsfragments/3691.minor b/newsfragments/3691.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index 50dde9e43..0c860e93a 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + try: from allmydata.scripts.types_ import SubCommands except ImportError: @@ -22,8 +24,10 @@ def print_keypair(options): from allmydata.crypto import ed25519 out = options.stdout private_key, public_key = ed25519.create_signing_keypair() - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), + file=out) + print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), + file=out) class DerivePubkeyOptions(BaseOptions): def parseArgs(self, privkey): @@ -45,9 +49,10 @@ def derive_pubkey(options): out = options.stdout from allmydata.crypto import ed25519 privkey_vs = options.privkey - private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) - print("private:", ed25519.string_from_signing_key(private_key), file=out) - print("public:", ed25519.string_from_verifying_key(public_key), file=out) + private_key, public_key = ed25519.signing_keypair_from_string( + privkey_vs.encode("ascii")) + print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out) + print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 class AdminCommand(BaseOptions): diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py index 71db3f5cb..b3e09b699 100644 --- a/src/allmydata/scripts/tahoe_ls.py +++ b/src/allmydata/scripts/tahoe_ls.py @@ -27,6 +27,8 @@ def list(options): except UnknownAliasError as e: e.display(stderr) return 1 + + path = unicode(path, "utf-8") url = nodeurl + "uri/%s" % url_quote(rootcap) if path: # move where.endswith check here? @@ -70,7 +72,7 @@ def list(options): children = d['children'] else: # paths returned from get_alias are always valid UTF-8 - childname = path.split("/")[-1].decode('utf-8') + childname = path.split("/")[-1] children = {childname: (nodetype, d)} if "metadata" not in d: d["metadata"] = {} diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py index bdaf134ff..84f83edcd 100644 --- a/src/allmydata/scripts/tahoe_mv.py +++ b/src/allmydata/scripts/tahoe_mv.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import re from urllib.parse import quote as url_quote import json @@ -25,6 +27,7 @@ def mv(options, mode="move"): except UnknownAliasError as e: e.display(stderr) return 1 + from_path = unicode(from_path, "utf-8") from_url = nodeurl + "uri/%s" % url_quote(rootcap) if from_path: from_url += "/" + escape_path(from_path) @@ -44,6 +47,7 @@ def mv(options, mode="move"): e.display(stderr) return 1 to_url = nodeurl + "uri/%s" % url_quote(rootcap) + path = unicode(path, "utf-8") if path: to_url += "/" + escape_path(path) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2b1bc1c86..d08d46a34 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os.path from six.moves import cStringIO as StringIO import urllib, sys @@ -361,11 +363,11 @@ class CLI(CLITestMixin, unittest.TestCase): "didn't see 'mqfblse6m5a6dh45isu2cg7oji' in '%s'" % err) def test_alias(self): - def s128(c): return base32.b2a(c*(128/8)) - def s256(c): return base32.b2a(c*(256/8)) - TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T")) - WA = "URI:DIR2:%s:%s" % (s128("W"), s256("W")) - CA = "URI:DIR2:%s:%s" % (s128("C"), s256("C")) + def s128(c): return base32.b2a(c*(128//8)) + def s256(c): return base32.b2a(c*(256//8)) + TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T")) + WA = b"URI:DIR2:%s:%s" % (s128(b"W"), s256(b"W")) + CA = b"URI:DIR2:%s:%s" % (s128(b"C"), s256(b"C")) aliases = {"tahoe": TA, "work": WA, "c": CA} @@ -718,8 +720,9 @@ class Admin(unittest.TestCase): self.failUnlessEqual(pubkey_bits[0], vk_header, lines[1]) self.failUnless(privkey_bits[1].startswith("priv-v0-"), lines[0]) self.failUnless(pubkey_bits[1].startswith("pub-v0-"), lines[1]) - sk, pk = ed25519.signing_keypair_from_string(privkey_bits[1]) - vk_bytes = pubkey_bits[1] + sk, pk = ed25519.signing_keypair_from_string( + privkey_bits[1].encode("ascii")) + vk_bytes = pubkey_bits[1].encode("ascii") self.assertEqual( ed25519.string_from_verifying_key(pk), vk_bytes, @@ -729,8 +732,8 @@ class Admin(unittest.TestCase): def test_derive_pubkey(self): priv_key, pub_key = ed25519.create_signing_keypair() - priv_key_str = ed25519.string_from_signing_key(priv_key) - pub_key_str = ed25519.string_from_verifying_key(pub_key) + priv_key_str = unicode(ed25519.string_from_signing_key(priv_key), "ascii") + pub_key_str = unicode(ed25519.string_from_verifying_key(pub_key), "ascii") d = run_cli("admin", "derive-pubkey", priv_key_str) def _done(args): (rc, stdout, stderr) = args diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py index fff57cdc9..1206579f1 100644 --- a/src/allmydata/test/cli/test_list.py +++ b/src/allmydata/test/cli/test_list.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_str + from twisted.trial import unittest from twisted.internet import defer @@ -8,61 +21,52 @@ from ..no_network import GridTestMixin from allmydata.util.encodingutil import quote_output, get_io_encoding from .common import CLITestMixin + class List(GridTestMixin, CLITestMixin, unittest.TestCase): def test_list(self): self.basedir = "cli/List/list" self.set_up_grid() c0 = self.g.clients[0] - small = "small" + small = b"small" - # u"g\u00F6\u00F6d" might not be representable in the argv and/or output encodings. - # It is initially included in the directory in any case. - try: - good_arg = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_arg = None + good_arg = u"g\u00F6\u00F6d" + good_out = u"g\u00F6\u00F6d" - try: - good_out = u"g\u00F6\u00F6d".encode(get_io_encoding()) - except UnicodeEncodeError: - good_out = None + # On Python 2 we get bytes, so we need encoded version. On Python 3 + # stdio is unicode so can leave unchanged. + good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding()) d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence="")) + self.rooturi = str(n.get_uri(), "utf-8") + return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_goodcap(n): self.goodcap = n.get_uri() d.addCallback(_stash_goodcap) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"1share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(1,10))) + self.delete_shares_numbered(n.get_uri(), list(range(1,10)))) d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"0share")) d.addCallback(lambda n: - self.delete_shares_numbered(n.get_uri(), range(0,10))) + self.delete_shares_numbered(n.get_uri(), list(range(0,10)))) d.addCallback(lambda ign: self.do_cli("add-alias", "tahoe", self.rooturi)) d.addCallback(lambda ign: self.do_cli("ls")) def _check1(args): (rc, out, err) = args - if good_out is None: - self.failUnlessReallyEqual(rc, 1) - self.failUnlessIn("files whose names could not be converted", err) - self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share"])) - else: - self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") - self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", good_out])) + self.failUnlessReallyEqual(rc, 0) + self.assertEqual(len(err), 0, err) + expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded]) + self.assertEqual(sorted(out.splitlines()), expected) d.addCallback(_check1) d.addCallback(lambda ign: self.do_cli("ls", "missing")) def _check2(args): (rc, out, err) = args self.failIfEqual(rc, 0) - self.failUnlessReallyEqual(err.strip(), "No such file or directory") - self.failUnlessReallyEqual(out, "") + self.assertEqual(err.strip(), "No such file or directory") + self.assertEqual(len(out), 0, out) d.addCallback(_check2) d.addCallback(lambda ign: self.do_cli("ls", "1share")) def _check3(args): @@ -72,7 +76,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessIn("UnrecoverableFileError:", err) self.failUnlessIn("could not be retrieved, because there were " "insufficient good shares.", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check3) d.addCallback(lambda ign: self.do_cli("ls", "0share")) d.addCallback(_check3) @@ -82,13 +86,13 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("files whose names could not be converted", err) self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) else: # listing a file (as dir/filename) should have the edge metadata, # including the filename self.failUnlessReallyEqual(rc, 0) - self.failUnlessIn(good_out, out) - self.failIfIn("-r-- %d -" % len(small), out, + self.failUnlessIn(good_out_encoded, out) + self.failIfIn(ensure_str("-r-- %d -" % len(small)), out, "trailing hyphen means unknown date") if good_arg is not None: @@ -106,7 +110,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # metadata, just the size (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip()) + self.assertEqual("-r-- %d -" % len(small), out.strip()) d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap)) d.addCallback(_check5) @@ -118,7 +122,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _check1_ascii(args): (rc,out,err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", "good"])) d.addCallback(_check1_ascii) def _check4_ascii(args): @@ -139,7 +143,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good")) d.addCallback(_check4_ascii) - unknown_immcap = "imm.URI:unknown" + unknown_immcap = b"imm.URI:unknown" def _create_unknown(ign): nm = c0.nodemaker kids = {u"unknownchild-imm": (nm.create_from_cap(unknown_immcap), {})} @@ -178,7 +182,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -193,7 +197,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -226,8 +230,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): # The uploaders may run at the same time, so we need two # MutableData instances or they'll fight over offsets &c and # break. - mutable_data = MutableData("data" * 100000) - mutable_data2 = MutableData("data" * 100000) + mutable_data = MutableData(b"data" * 100000) + mutable_data2 = MutableData(b"data" * 100000) # Add both kinds of mutable node. d1 = nm.create_mutable_file(mutable_data, version=MDMF_VERSION) @@ -235,8 +239,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): version=SDMF_VERSION) # Add an immutable node. We do this through the directory, # with add_file. - immutable_data = upload.Data("immutable data" * 100000, - convergence="") + immutable_data = upload.Data(b"immutable data" * 100000, + convergence=b"") d3 = n.add_file(u"immutable", immutable_data) ds = [d1, d2, d3] dl = defer.DeferredList(ds) @@ -294,12 +298,12 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase): def _got_json(args): (rc, out, err) = args self.failUnlessEqual(rc, 0) - self.failUnlessEqual(err, "") - self.failUnlessIn(self._mdmf_uri, out) - self.failUnlessIn(self._mdmf_readonly_uri, out) - self.failUnlessIn(self._sdmf_uri, out) - self.failUnlessIn(self._sdmf_readonly_uri, out) - self.failUnlessIn(self._imm_uri, out) + self.assertEqual(len(err), 0, err) + self.failUnlessIn(str(self._mdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._mdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_uri, "ascii"), out) + self.failUnlessIn(str(self._sdmf_readonly_uri, "ascii"), out) + self.failUnlessIn(str(self._imm_uri, "ascii"), out) self.failUnlessIn('"format": "SDMF"', out) self.failUnlessIn('"format": "MDMF"', out) d.addCallback(_got_json) diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py index 9d1a64974..0bb9ba369 100644 --- a/src/allmydata/test/cli/test_mv.py +++ b/src/allmydata/test/cli/test_mv.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from allmydata.util import fileutil @@ -5,15 +17,16 @@ from ..no_network import GridTestMixin from allmydata.scripts import tahoe_mv from .common import CLITestMixin + class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): def test_mv_behavior(self): self.basedir = "cli/Mv/mv_behavior" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) fn2 = os.path.join(self.basedir, "file2") - DATA2 = "UML diagrams" + DATA2 = b"UML diagrams" fileutil.write(fn2, DATA2) # copy both files to the grid d = self.do_cli("create-alias", "tahoe") @@ -104,11 +117,11 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Mv/mv_error_if_DELETE_fails" self.set_up_grid(oneshare=True) fn1 = os.path.join(self.basedir, "file1") - DATA1 = "Nuclear launch codes" + DATA1 = b"Nuclear launch codes" fileutil.write(fn1, DATA1) original_do_http = tahoe_mv.do_http - def mock_do_http(method, url, body=""): + def mock_do_http(method, url, body=b""): if method == "DELETE": class FakeResponse(object): def read(self): @@ -152,7 +165,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first @@ -180,7 +193,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("fake", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) # check to see that the validation extends to the # target argument by making an alias that will work with the first diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index d27791f34..6100d2568 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -1,6 +1,16 @@ """ Tests for ``allmydata.scripts.tahoe_run``. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import ( StringIO, @@ -50,7 +60,7 @@ class DaemonizeTheRealServiceTests(SyncTestCase): """ nodedir = FilePath(self.mktemp()) nodedir.makedirs() - nodedir.child("tahoe.cfg").setContent(config) + nodedir.child("tahoe.cfg").setContent(config.encode("ascii")) nodedir.child("tahoe-client.tac").touch() options = parse_options(["run", nodedir.path]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6695b685f..bbafc144f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -181,7 +181,10 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_create", "allmydata.test.cli.test_create_alias", "allmydata.test.cli.test_invite", + "allmydata.test.cli.test_list", + "allmydata.test.cli.test_mv", "allmydata.test.cli.test_put", + "allmydata.test.cli.test_run", "allmydata.test.cli.test_status", "allmydata.test.mutable.test_checker",