From 83e16d40a452ac5766d07094243c497f37a3a624 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:18:36 -0400 Subject: [PATCH 01/10] Some tests passing on Python 3. --- src/allmydata/scripts/cli.py | 6 +++--- src/allmydata/scripts/tahoe_check.py | 8 ++++---- src/allmydata/test/cli/test_check.py | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 811ae7ef9..826c36e1f 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions): def parseArgs(self, *args): if len(args) < 2: raise usage.UsageError("cp requires at least two arguments") - self.sources = map(argv_to_unicode, args[:-1]) + self.sources = list(map(argv_to_unicode, args[:-1])) self.destination = argv_to_unicode(args[-1]) synopsis = "[options] FROM.. TO" @@ -435,7 +435,7 @@ class CheckOptions(FileStoreOptions): ("add-lease", None, "Add/renew lease on all shares."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ @@ -452,7 +452,7 @@ class DeepCheckOptions(FileStoreOptions): ("verbose", "v", "Be noisy about what is happening."), ] def parseArgs(self, *locations): - self.locations = map(argv_to_unicode, locations) + self.locations = list(map(argv_to_unicode, locations)) synopsis = "[options] [ALIAS:PATH]" description = """ diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index cef9e32be..1700e2b77 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,6 +1,6 @@ from __future__ import print_function -import urllib +from urllib.parse import quote as url_quote import json # Python 2 compatibility @@ -36,7 +36,7 @@ def check_location(options, where): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -139,7 +139,7 @@ class DeepCheckOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -297,7 +297,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 8cf963da6..eef8d3108 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -18,7 +18,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.basedir = "cli/Check/check" self.set_up_grid() c0 = self.g.clients[0] - DATA = "data" * 100 + DATA = b"data" * 100 DATA_uploadable = MutableData(DATA) d = c0.create_mutable_file(DATA_uploadable) def _stash_uri(n): @@ -45,7 +45,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) - d.addCallback(lambda ign: c0.upload(upload.Data("literal", convergence=""))) + d.addCallback(lambda ign: c0.upload(upload.Data(b"literal", convergence=b""))) def _stash_lit_uri(n): self.lit_uri = n.get_uri() d.addCallback(_stash_lit_uri) @@ -156,14 +156,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 quoted_good = quote_output(u"g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n self.rooturi = n.get_uri() - return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence="")) + return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -171,11 +171,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_stash_uri, u"g\u00F6\u00F6d") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", - convergence=""))) + upload.Data(b"literal", + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: - c0.create_mutable_file(MutableData(DATA+"1"))) + c0.create_mutable_file(MutableData(DATA+b"1"))) d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn)) d.addCallback(_stash_uri, "mutable") @@ -322,7 +322,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir")) d.addCallback(_stash_uri, "subdir") d.addCallback(lambda fn: - fn.add_file(u"subfile", upload.Data(DATA+"2", ""))) + fn.add_file(u"subfile", upload.Data(DATA+b"2", b""))) d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], range(10))) From 5e59b9d8d6a059fd7f14129d7bf8785aa39a4fed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:30:37 -0400 Subject: [PATCH 02/10] A little closer to passing tests on Python 3. --- src/allmydata/scripts/slow_operation.py | 4 ++-- src/allmydata/scripts/tahoe_check.py | 10 +++++++--- src/allmydata/test/cli/test_check.py | 15 ++++++++------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index ce25e9667..cce7d91c1 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -6,7 +6,7 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ from allmydata.scripts.common_http import do_http, format_http_error from allmydata.util import base32 from allmydata.util.encodingutil import quote_output, is_printable_ascii -import urllib +from urllib.parse import quote as url_quote import json class SlowOperationRunner(object): @@ -27,7 +27,7 @@ class SlowOperationRunner(object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 1700e2b77..08569ec5d 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote import json # Python 2 compatibility -from future.utils import PY2 +from future.utils import PY2, PY3 if PY2: from future.builtins import str # noqa: F401 @@ -54,8 +54,12 @@ def check_location(options, where): return 1 jdata = resp.read() if options.get("raw"): - stdout.write(jdata) - stdout.write("\n") + if PY3: + stdoutb = stdout.buffer + else: + stdoutb = stdout + stdoutb.write(jdata) + stdoutb.write(b"\n") return 0 data = json.loads(jdata) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index eef8d3108..e1a45faa2 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,3 +1,4 @@ +from past.builtins import unicode import os.path import json from twisted.trial import unittest @@ -41,7 +42,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(err, "") self.failUnlessReallyEqual(rc, 0) data = json.loads(out) - self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy") + self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy") self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check2) @@ -68,7 +69,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(data["results"]["healthy"], True) d.addCallback(_check_lit_raw) - d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence="")) + d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=b"")) def _stash_lit_dir_uri(n): self.lit_dir_uri = n.get_uri() d.addCallback(_stash_lit_dir_uri) @@ -89,9 +90,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), - shares[1][0]) + (unicode(base32.b2a(shares[1][1]), "ascii"), + unicode(base32.b2a(storage_index), "ascii"), + shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -418,8 +419,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 0) self.failUnlessReallyEqual(err, "") #Ensure healthy appears for each uri - self.failUnlessIn("Healthy", out[:len(out)/2]) - self.failUnlessIn("Healthy", out[len(out)/2:]) + self.failUnlessIn("Healthy", out[:len(out)//2]) + self.failUnlessIn("Healthy", out[len(out)//2:]) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], "nonexistent:")) From f6e0611b07c0e939fa96a27fa3d08a110b856a69 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:42:05 -0400 Subject: [PATCH 03/10] All tests pass on Python 3. --- src/allmydata/scripts/slow_operation.py | 3 ++- src/allmydata/scripts/tahoe_check.py | 9 +++++++-- src/allmydata/scripts/tahoe_manifest.py | 7 ++++--- src/allmydata/test/cli/test_check.py | 4 ++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index cce7d91c1..f7366fcc0 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,4 +1,5 @@ from __future__ import print_function +from six import ensure_str import os, time from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ @@ -14,7 +15,7 @@ class SlowOperationRunner(object): def run(self, options): stderr = options.stderr self.options = options - self.ophandle = ophandle = base32.b2a(os.urandom(16)) + self.ophandle = ophandle = ensure_str(base32.b2a(os.urandom(16))) nodeurl = options['node-url'] if not nodeurl.endswith("/"): nodeurl += "/" diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 08569ec5d..3859f0061 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -206,7 +206,7 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=self.stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.streamer.rc = 1 print(quote_output(line, quotemarks=False), file=self.stderr) @@ -321,12 +321,17 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 # use Twisted to split this into lines + if PY3: + stdoutb = stdout.buffer + else: + stdoutb = stdout + while True: chunk = resp.read(100) if not chunk: break if self.options["raw"]: - stdout.write(chunk) + stdoutb.write(chunk) else: output.dataReceived(chunk) if not self.options["raw"]: diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 386cdd1ad..6166564d3 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,6 +1,7 @@ from __future__ import print_function -import urllib, json +from urllib.parse import quote as url_quote +import json from twisted.protocols.basic import LineOnlyReceiver from allmydata.util.abbreviate import abbreviate_space_both from allmydata.scripts.slow_operation import SlowOperationRunner @@ -35,7 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object): return 1 if path == '/': path = '' - url = nodeurl + "uri/%s" % urllib.quote(rootcap) + url = nodeurl + "uri/%s" % url_quote(rootcap) if path: url += "/" + escape_path(path) # todo: should it end with a slash? @@ -63,7 +64,7 @@ class ManifestStreamer(LineOnlyReceiver, object): if self.in_error: print(quote_output(line, quotemarks=False), file=stderr) return - if line.startswith("ERROR:"): + if line.startswith(b"ERROR:"): self.in_error = True self.rc = 1 print(quote_output(line, quotemarks=False), file=stderr) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index e1a45faa2..756ed4f17 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -237,8 +237,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ - (base32.b2a(shares[1][1]), - base32.b2a(storage_index), + (unicode(base32.b2a(shares[1][1]), "ascii"), + unicode(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) From 87f1620ab0a6f7bce4da9fe9c6bb8bd02274df53 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:42:23 -0400 Subject: [PATCH 04/10] News file. --- newsfragments/3678.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3678.minor diff --git a/newsfragments/3678.minor b/newsfragments/3678.minor new file mode 100644 index 000000000..e69de29bb From 5ebb385c10f801fd72dc2248e748c3dcb1a69c65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 21 Apr 2021 11:58:48 -0400 Subject: [PATCH 05/10] Port to Python 3. --- src/allmydata/test/cli/test_check.py | 57 ++++++++++++++++------------ src/allmydata/util/_python3.py | 1 + 2 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 756ed4f17..fda8b4352 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -1,4 +1,12 @@ -from past.builtins import unicode +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path import json from twisted.trial import unittest @@ -13,6 +21,7 @@ from allmydata.scripts import debug from ..no_network import GridTestMixin from .common import CLITestMixin + class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def test_check(self): @@ -29,7 +38,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.uri)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy" in lines, out) @@ -39,7 +48,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy") @@ -54,7 +63,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", self.lit_uri)) def _check_lit(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: Healthy (LIT)" in lines, out) @@ -63,7 +72,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri)) def _check_lit_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], True) @@ -90,8 +99,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uri).get_storage_index() self._corrupt_share_line = " server %s, SI %s, shnum %d" % \ - (unicode(base32.b2a(shares[1][1]), "ascii"), - unicode(base32.b2a(storage_index), "ascii"), + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -99,7 +108,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() summary = [l for l in lines if l.startswith("Summary")][0] @@ -113,7 +122,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("check", "--verify", "--raw", self.uri)) def _check3_raw(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) data = json.loads(out) self.failUnlessReallyEqual(data["results"]["healthy"], False) @@ -127,7 +136,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: not healthy" in lines, out) @@ -141,7 +150,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("check", "--verify", "--repair", self.uri)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("Summary: healthy" in lines, out) @@ -183,7 +192,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) def _check1(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy" @@ -199,7 +208,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check2(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -213,7 +222,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(lambda ign: self.do_cli("stats", self.rooturi)) def _check_stats(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnlessIn(" count-immutable-files: 1", lines) @@ -237,8 +246,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): cso.parseOptions([shares[1][2]]) storage_index = uri.from_string(self.uris["mutable"]).get_storage_index() self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \ - (unicode(base32.b2a(shares[1][1]), "ascii"), - unicode(base32.b2a(storage_index), "ascii"), + (str(base32.b2a(shares[1][1]), "ascii"), + str(base32.b2a(storage_index), "ascii"), shares[1][0]) debug.corrupt_share(cso) d.addCallback(_clobber_shares) @@ -252,7 +261,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.do_cli("deep-check", "--verbose", self.rooturi)) def _check3(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -269,7 +278,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check4(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) @@ -288,7 +297,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check5(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() units = [json.loads(line) for line in lines] @@ -302,7 +311,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.rooturi)) def _check6(args): (rc, out, err) = args - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) lines = out.splitlines() self.failUnless("'': healthy" in lines, out) @@ -326,7 +335,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): fn.add_file(u"subfile", upload.Data(DATA+b"2", b""))) d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], - range(10))) + list(range(10)))) # root # rootg\u00F6\u00F6d/ @@ -380,7 +389,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) d.addCallback(lambda ign: self.do_cli("deep-check")) d.addCallback(_check) @@ -397,7 +406,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failUnlessReallyEqual(rc, 1) self.failUnlessIn("error:", err) self.failUnlessIn("nonexistent", err) - self.failUnlessReallyEqual(out, "") + self.assertEqual(len(out), 0, out) d.addCallback(_check) return d @@ -417,7 +426,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): def _check(args): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) - self.failUnlessReallyEqual(err, "") + self.assertEqual(len(err), 0, err) #Ensure healthy appears for each uri self.failUnlessIn("Healthy", out[:len(out)//2]) self.failUnlessIn("Healthy", out[len(out)//2:]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 3003d2909..26f6b8e2a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -176,6 +176,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.cli.test_alias", "allmydata.test.cli.test_backup", "allmydata.test.cli.test_backupdb", + "allmydata.test.cli.test_check", "allmydata.test.cli.test_create", "allmydata.test.cli.test_invite", "allmydata.test.cli.test_status", From 86fe350bef7a340d28f5947b289de6a2a927b2fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:15:43 -0400 Subject: [PATCH 06/10] Tests pass on Python 2. --- src/allmydata/test/cli/test_check.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index fda8b4352..320106be9 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -6,6 +6,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text import os.path import json @@ -167,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.uris = {} self.fileurls = {} DATA = b"data" * 100 - quoted_good = quote_output(u"g\u00F6\u00F6d") + quoted_good = u"'g\u00F6\u00F6d'" d = c0.create_dirnode() def _stash_root_and_create_file(n): @@ -210,6 +211,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -263,6 +265,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -280,6 +283,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': Healthy" in lines, out) self.failUnless("'small': Healthy (LIT)" in lines, out) @@ -313,6 +317,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.assertEqual(len(err), 0, err) self.failUnlessReallyEqual(rc, 0) + out = ensure_text(out) lines = out.splitlines() self.failUnless("'': healthy" in lines, out) self.failUnless("'small': healthy" in lines, out) @@ -350,7 +355,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.failIfEqual(rc, 0) self.failUnlessIn("ERROR: UnrecoverableFileError", err) # the fatal directory should still show up, as the last line - self.failUnlessIn(" subdir\n", out) + self.failUnlessIn(" subdir\n", ensure_text(out)) d.addCallback(_manifest_failed) d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi)) From b675ca23800fa52f7cb8ad2192952a0565587a2d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:23:58 -0400 Subject: [PATCH 07/10] Lint fix. --- src/allmydata/test/cli/test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index 320106be9..e01dcc4cb 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import quote_output, to_bytes +from allmydata.util.encodingutil import to_bytes from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug From a393b54315e6f46bc82b2b3781125487b26215a2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 22 Apr 2021 10:27:59 -0400 Subject: [PATCH 08/10] Fix BytesWarning errors. --- src/allmydata/scripts/slow_operation.py | 3 +++ src/allmydata/scripts/tahoe_check.py | 2 ++ src/allmydata/scripts/tahoe_manifest.py | 3 +++ 3 files changed, 8 insertions(+) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index f7366fcc0..5d8c77538 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,4 +1,6 @@ from __future__ import print_function + +from past.builtins import unicode from six import ensure_str import os, time @@ -26,6 +28,7 @@ class SlowOperationRunner(object): except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 3859f0061..d8b7c9bce 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -34,6 +34,7 @@ def check_location(options, where): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) @@ -299,6 +300,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py index 6166564d3..b837e648a 100644 --- a/src/allmydata/scripts/tahoe_manifest.py +++ b/src/allmydata/scripts/tahoe_manifest.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + from urllib.parse import quote as url_quote import json from twisted.protocols.basic import LineOnlyReceiver @@ -34,6 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object): except UnknownAliasError as e: e.display(stderr) return 1 + path = unicode(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) From 9137da5483c9dddc62f3e7ab639151c3d4472c44 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 10:16:41 -0400 Subject: [PATCH 09/10] Stick to Unicode when possible. --- src/allmydata/scripts/tahoe_check.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index d8b7c9bce..494c1dba3 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -53,14 +53,11 @@ def check_location(options, where): if resp.status != 200: print(format_http_error("ERROR", resp), file=stderr) return 1 - jdata = resp.read() + jdata = resp.read().decode() + if options.get("raw"): - if PY3: - stdoutb = stdout.buffer - else: - stdoutb = stdout - stdoutb.write(jdata) - stdoutb.write(b"\n") + stdout.write(jdata) + stdout.write("\n") return 0 data = json.loads(jdata) @@ -323,17 +320,12 @@ class DeepCheckStreamer(LineOnlyReceiver, object): return 1 # use Twisted to split this into lines - if PY3: - stdoutb = stdout.buffer - else: - stdoutb = stdout - while True: chunk = resp.read(100) if not chunk: break if self.options["raw"]: - stdoutb.write(chunk) + stdout.write(chunk.decode()) else: output.dataReceived(chunk) if not self.options["raw"]: From 72a85ba62422b3e398459bde3ea6ad8cbd2fafef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Apr 2021 10:19:59 -0400 Subject: [PATCH 10/10] Fix lint. --- src/allmydata/scripts/tahoe_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 494c1dba3..82885d073 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote import json # Python 2 compatibility -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401