From b24a9f70832984a942a017a3190cd95c0df75df2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Dec 2020 11:21:04 -0500 Subject: [PATCH 01/27] Trying to get test_grid working on Python 3. --- src/allmydata/test/no_network.py | 1 - src/allmydata/test/web/test_grid.py | 123 ++++++++++++++-------------- src/allmydata/web/common.py | 9 +- src/allmydata/web/common_py3.py | 8 ++ src/allmydata/web/filenode.py | 14 ++-- src/allmydata/web/root.py | 5 +- src/allmydata/webish.py | 3 +- 7 files changed, 89 insertions(+), 74 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 59ab807bb..9a2713830 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -614,7 +614,6 @@ class GridTestMixin(object): method="GET", clientnum=0, **kwargs): # if return_response=True, this fires with (data, statuscode, # respheaders) instead of just data. - assert not isinstance(urlpath, unicode) url = self.client_baseurls[clientnum] + urlpath response = yield treq.request(method, url, persistent=False, diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 8f61781d4..04c3edbac 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1,6 +1,7 @@ from __future__ import print_function -import os.path, re, urllib +import os.path, re +from urllib.parse import quote as url_quote import json from six.moves import StringIO @@ -37,7 +38,7 @@ DIR_HTML_TAG = '' class CompletelyUnhandledError(Exception): pass -class ErrorBoom(object, resource.Resource): +class ErrorBoom(resource.Resource): @render_exception def render(self, req): raise CompletelyUnhandledError("whoops") @@ -54,25 +55,25 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.set_up_grid() c0 = self.g.clients[0] self.uris = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA, convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_uri(ur, which): self.uris[which] = ur.get_uri() d.addCallback(_stash_uri, "good") d.addCallback(lambda ign: - c0.upload(upload.Data(DATA+"1", convergence=""))) + c0.upload(upload.Data(DATA+b"1", convergence=b""))) d.addCallback(_stash_uri, "sick") d.addCallback(lambda ign: - c0.upload(upload.Data(DATA+"2", convergence=""))) + c0.upload(upload.Data(DATA+b"2", convergence=b""))) d.addCallback(_stash_uri, "dead") def _stash_mutable_uri(n, which): self.uris[which] = n.get_uri() - assert isinstance(self.uris[which], str) + assert isinstance(self.uris[which], bytes) d.addCallback(lambda ign: - c0.create_mutable_file(publish.MutableData(DATA+"3"))) + c0.create_mutable_file(publish.MutableData(DATA+b"3"))) d.addCallback(_stash_mutable_uri, "corrupt") d.addCallback(lambda ign: - c0.upload(upload.Data("literal", convergence=""))) + c0.upload(upload.Data("literal", convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: c0.create_immutable_dirnode({})) d.addCallback(_stash_mutable_uri, "smalldir") @@ -80,7 +81,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _compute_fileurls(ignored): self.fileurls = {} for which in self.uris: - self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) + self.fileurls[which] = "uri/" + url_quote(self.uris[which]) d.addCallback(_compute_fileurls) def _clobber_shares(ignored): @@ -203,28 +204,28 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.set_up_grid() c0 = self.g.clients[0] self.uris = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA, convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_uri(ur, which): self.uris[which] = ur.get_uri() d.addCallback(_stash_uri, "good") d.addCallback(lambda ign: - c0.upload(upload.Data(DATA+"1", convergence=""))) + c0.upload(upload.Data(DATA+b"1", convergence=b""))) d.addCallback(_stash_uri, "sick") d.addCallback(lambda ign: - c0.upload(upload.Data(DATA+"2", convergence=""))) + c0.upload(upload.Data(DATA+b"2", convergence=b""))) d.addCallback(_stash_uri, "dead") def _stash_mutable_uri(n, which): self.uris[which] = n.get_uri() - assert isinstance(self.uris[which], str) + assert isinstance(self.uris[which], bytes) d.addCallback(lambda ign: - c0.create_mutable_file(publish.MutableData(DATA+"3"))) + c0.create_mutable_file(publish.MutableData(DATA+b"3"))) d.addCallback(_stash_mutable_uri, "corrupt") def _compute_fileurls(ignored): self.fileurls = {} for which in self.uris: - self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) + self.fileurls[which] = "uri/" + url_quote(self.uris[which]) d.addCallback(_compute_fileurls) def _clobber_shares(ignored): @@ -286,8 +287,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.set_up_grid() c0 = self.g.clients[0] self.uris = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA+"1", convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA+b"1", convergence=b"")) def _stash_uri(ur, which): self.uris[which] = ur.get_uri() d.addCallback(_stash_uri, "sick") @@ -295,7 +296,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _compute_fileurls(ignored): self.fileurls = {} for which in self.uris: - self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) + self.fileurls[which] = "uri/" + url_quote(self.uris[which]) d.addCallback(_compute_fileurls) def _clobber_shares(ignored): @@ -329,7 +330,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.fileurls = {} # the future cap format may contain slashes, which must be tolerated - expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap, + expected_info_url = "uri/%s?t=info" % url_quote(unknown_rwcap, safe="") if immutable: @@ -343,8 +344,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _stash_root_and_create_file(n): self.rootnode = n - self.rooturl = "uri/" + urllib.quote(n.get_uri()) - self.rourl = "uri/" + urllib.quote(n.get_readonly_uri()) + self.rooturl = "uri/" + url_quote(n.get_uri()) + self.rourl = "uri/" + url_quote(n.get_readonly_uri()) if not immutable: return self.rootnode.set_node(name, future_node) d.addCallback(_stash_root_and_create_file) @@ -510,7 +511,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("CHK", cap.to_string()) self.cap = cap self.rootnode = dn - self.rooturl = "uri/" + urllib.quote(dn.get_uri()) + self.rooturl = "uri/" + url_quote(dn.get_uri()) return download_to_data(dn._node) d.addCallback(_created) @@ -559,7 +560,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if td.text != u"FILE": continue a = td.findNextSibling()(u"a")[0] - self.assertIn(urllib.quote(lonely_uri), a[u"href"]) + self.assertIn(url_quote(lonely_uri), a[u"href"]) self.assertEqual(u"lonely", a.text) self.assertEqual(a[u"rel"], [u"noreferrer"]) self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text) @@ -573,7 +574,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi if a.text == u"More Info" ) self.assertEqual(1, len(infos)) - self.assertTrue(infos[0].endswith(urllib.quote(lonely_uri) + "?t=info")) + self.assertTrue(infos[0].endswith(url_quote(lonely_uri) + "?t=info")) d.addCallback(_check_html) # ... and in JSON. @@ -596,12 +597,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) - return n.add_file(u"good", upload.Data(DATA, convergence="")) + self.fileurls["root"] = "uri/" + url_quote(n.get_uri()) + return n.add_file(u"good", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -610,12 +611,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.rootnode.add_file(u"small", upload.Data("literal", - convergence=""))) + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: self.rootnode.add_file(u"sick", - upload.Data(DATA+"1", - convergence=""))) + upload.Data(DATA+b"1", + convergence=b""))) d.addCallback(_stash_uri, "sick") # this tests that deep-check and stream-manifest will ignore @@ -695,8 +696,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_uri, "subdir") d.addCallback(lambda subdir_node: subdir_node.add_file(u"grandchild", - upload.Data(DATA+"2", - convergence=""))) + upload.Data(DATA+b"2", + convergence=b""))) d.addCallback(_stash_uri, "grandchild") d.addCallback(lambda ign: @@ -770,12 +771,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) - return n.add_file(u"good", upload.Data(DATA, convergence="")) + self.fileurls["root"] = "uri/" + url_quote(n.get_uri()) + return n.add_file(u"good", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -783,17 +784,17 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.rootnode.add_file(u"small", upload.Data("literal", - convergence=""))) + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: self.rootnode.add_file(u"sick", - upload.Data(DATA+"1", - convergence=""))) + upload.Data(DATA+b"1", + convergence=b""))) d.addCallback(_stash_uri, "sick") #d.addCallback(lambda ign: # self.rootnode.add_file(u"dead", - # upload.Data(DATA+"2", - # convergence=""))) + # upload.Data(DATA+b"2", + # convergence=b""))) #d.addCallback(_stash_uri, "dead") #d.addCallback(lambda ign: c0.create_mutable_file("mutable")) @@ -888,25 +889,25 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.set_up_grid(num_clients=2, oneshare=True) c0 = self.g.clients[0] self.uris = {} - DATA = "data" * 100 - d = c0.upload(upload.Data(DATA, convergence="")) + DATA = b"data" * 100 + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_uri(ur, which): self.uris[which] = ur.get_uri() d.addCallback(_stash_uri, "one") d.addCallback(lambda ign: - c0.upload(upload.Data(DATA+"1", convergence=""))) + c0.upload(upload.Data(DATA+b"1", convergence=b""))) d.addCallback(_stash_uri, "two") def _stash_mutable_uri(n, which): self.uris[which] = n.get_uri() - assert isinstance(self.uris[which], str) + assert isinstance(self.uris[which], bytes) d.addCallback(lambda ign: - c0.create_mutable_file(publish.MutableData(DATA+"2"))) + c0.create_mutable_file(publish.MutableData(DATA+b"2"))) d.addCallback(_stash_mutable_uri, "mutable") def _compute_fileurls(ignored): self.fileurls = {} for which in self.uris: - self.fileurls[which] = "uri/" + urllib.quote(self.uris[which]) + self.fileurls[which] = "uri/" + url_quote(self.uris[which]) d.addCallback(_compute_fileurls) d.addCallback(self._count_leases, "one") @@ -982,13 +983,13 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0 = self.g.clients[0] self.uris = {} self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 d = c0.create_dirnode() def _stash_root_and_create_file(n): self.rootnode = n self.uris["root"] = n.get_uri() - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) - return n.add_file(u"one", upload.Data(DATA, convergence="")) + self.fileurls["root"] = "uri/" + url_quote(n.get_uri()) + return n.add_file(u"one", upload.Data(DATA, convergence=b"")) d.addCallback(_stash_root_and_create_file) def _stash_uri(fn, which): self.uris[which] = fn.get_uri() @@ -996,7 +997,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.rootnode.add_file(u"small", upload.Data("literal", - convergence=""))) + convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: @@ -1051,34 +1052,34 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0 = self.g.clients[0] c0.encoding_params['happy'] = 2 self.fileurls = {} - DATA = "data" * 100 + DATA = b"data" * 100 d = c0.create_dirnode() def _stash_root(n): - self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri()) + self.fileurls["root"] = "uri/" + url_quote(n.get_uri()) self.fileurls["imaginary"] = self.fileurls["root"] + "/imaginary" return n d.addCallback(_stash_root) - d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence=""))) + d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence=b""))) def _stash_bad(ur): - self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri()) + self.fileurls["1share"] = "uri/" + url_quote(ur.get_uri()) self.delete_shares_numbered(ur.get_uri(), range(1,10)) u = uri.from_string(ur.get_uri()) u.key = testutil.flip_bit(u.key, 0) baduri = u.to_string() - self.fileurls["0shares"] = "uri/" + urllib.quote(baduri) + self.fileurls["0shares"] = "uri/" + url_quote(baduri) d.addCallback(_stash_bad) d.addCallback(lambda ign: c0.create_dirnode()) def _mangle_dirnode_1share(n): u = n.get_uri() - url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u) + url = self.fileurls["dir-1share"] = "uri/" + url_quote(u) self.fileurls["dir-1share-json"] = url + "?t=json" self.delete_shares_numbered(u, range(1,10)) d.addCallback(_mangle_dirnode_1share) d.addCallback(lambda ign: c0.create_dirnode()) def _mangle_dirnode_0share(n): u = n.get_uri() - url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u) + url = self.fileurls["dir-0share"] = "uri/" + url_quote(u) self.fileurls["dir-0share-json"] = url + "?t=json" self.delete_shares_numbered(u, range(0,10)) d.addCallback(_mangle_dirnode_0share) @@ -1269,9 +1270,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0 = self.g.clients[0] fn = c0.config.get_config_path("access.blacklist") self.uris = {} - DATA = "off-limits " * 50 + DATA = b"off-limits " * 50 - d = c0.upload(upload.Data(DATA, convergence="")) + d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_uri_and_create_dir(ur): self.uri = ur.get_uri() self.url = "uri/"+self.uri diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 4e8cab89d..98f915a1e 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,4 +1,5 @@ from past.builtins import unicode +from six import ensure_text import time import json @@ -99,11 +100,13 @@ def get_filenode_metadata(filenode): def boolean_of_arg(arg): # TODO: "" + arg = ensure_text(arg) if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"): raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST) return arg.lower() in ("true", "t", "1", "on") def parse_replace_arg(replace): + replace = ensure_text(replace) if replace.lower() == "only-files": return replace try: @@ -118,11 +121,11 @@ def get_format(req, default="CHK"): if boolean_of_arg(get_arg(req, "mutable", "false")): return "SDMF" return default - if arg.upper() == "CHK": + if arg.upper() == b"CHK": return "CHK" - elif arg.upper() == "SDMF": + elif arg.upper() == b"SDMF": return "SDMF" - elif arg.upper() == "MDMF": + elif arg.upper() == b"MDMF": return "MDMF" else: raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg, diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 89efd4d5d..05fc2b50b 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -4,6 +4,8 @@ Common utilities that are available from Python 3. Can eventually be merged back into allmydata.web.common. """ +from past.builtins import unicode + from twisted.web import resource, http from allmydata.util import abbreviate @@ -23,7 +25,13 @@ def get_arg(req, argname, default=None, multiple=False): empty), starting with all those in the query args. :param TahoeLAFSRequest req: The request to consider. + + :return: Either bytes or tuple of bytes. """ + if isinstance(argname, unicode): + argname = argname.encode("ascii") + if isinstance(default, unicode): + default = default.encode("ascii") results = [] if argname in req.args: results.extend(req.args[argname]) diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index f65977460..d90449e9f 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -133,9 +133,9 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin): @render_exception def render_POST(self, req): - t = get_arg(req, "t", "").strip() - replace = boolean_of_arg(get_arg(req, "replace", "true")) - if t == "upload": + t = get_arg(req, b"t", b"").strip() + replace = boolean_of_arg(get_arg(req, b"replace", b"true")) + if t == b"upload": # like PUT, but get the file data from an HTML form's input field. # We could get here from POST /uri/mutablefilecap?t=upload, # or POST /uri/path/file?t=upload, or @@ -290,11 +290,11 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object): @render_exception def render_POST(self, req): - t = get_arg(req, "t", "").strip() - replace = boolean_of_arg(get_arg(req, "replace", "true")) - if t == "check": + t = get_arg(req, b"t", b"").strip() + replace = boolean_of_arg(get_arg(req, b"replace", b"true")) + if t == b"check": d = self._POST_check(req) - elif t == "upload": + elif t == b"upload": # like PUT, but get the file data from an HTML form's input field # We could get here from POST /uri/mutablefilecap?t=upload, # or POST /uri/path/file?t=upload, or diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index cb5ddc070..5d45d41b3 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -226,7 +226,10 @@ class Root(MultiFormatResource): self._client = client self._now_fn = now_fn - self.putChild("uri", URIHandler(client)) + # Children need to be bytes; for now just doing these to make specific + # tests pass on Python 3, but eventually will do all them when this + # module is ported to Python 3 (if not earlier). + self.putChild(b"uri", URIHandler(client)) self.putChild("cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index b5e310fbc..a7f44514f 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -110,6 +110,7 @@ def _get_client_ip(request): def _logFormatter(logDateTime, request): + print("REQUEST: {}".format(request.uri)) # we build up a log string that hides most of the cap, to preserve # user privacy. We retain the query args so we can identify things # like t=json. Then we send it to the flog. We make no attempt to @@ -128,7 +129,7 @@ def _logFormatter(logDateTime, request): # sure we censor these too. if queryargs.startswith(b"uri="): queryargs = b"uri=[CENSORED]" - queryargs = "?" + queryargs + queryargs = b"?" + queryargs if path.startswith(b"/uri/"): path = b"/uri/[CENSORED]" elif path.startswith(b"/file/"): From c71acf93fd056e1ea9ff6e6ac7d2094ceaedb221 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Dec 2020 16:10:23 -0500 Subject: [PATCH 02/27] Bytes, alas. --- src/allmydata/web/filenode.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index d90449e9f..b58cc71f9 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,3 +1,4 @@ +from past.builtins import unicode import json @@ -117,7 +118,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin): @render_exception def render_PUT(self, req): - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() replace = parse_replace_arg(get_arg(req, "replace", "true")) assert self.parentnode and self.name @@ -179,7 +180,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object): @render_exception def render_GET(self, req): - t = get_arg(req, "t", "").strip() + t = unicode(get_arg(req, b"t", b"").strip(), "ascii") # t=info contains variable ophandles, so is not allowed an ETag. FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"] @@ -237,19 +238,19 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object): @render_exception def render_HEAD(self, req): - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() if t: raise WebError("HEAD file: bad t=%s" % t) - filename = get_arg(req, "filename", self.name) or "unknown" + filename = get_arg(req, b"filename", self.name) or "unknown" d = self.node.get_best_readable_version() d.addCallback(lambda dn: FileDownloader(dn, filename)) return d @render_exception def render_PUT(self, req): - t = get_arg(req, "t", "").strip() - replace = parse_replace_arg(get_arg(req, "replace", "true")) - offset = parse_offset_arg(get_arg(req, "offset", None)) + t = get_arg(req, b"t", b"").strip() + replace = parse_replace_arg(get_arg(req, b"replace", b"true")) + offset = parse_offset_arg(get_arg(req, b"offset", None)) if not t: if not replace: From 2ec7d52d09a0a44f3e76ba1ee40aeb24852431af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 18 Dec 2020 16:10:31 -0500 Subject: [PATCH 03/27] Some progress towards passing tests on Python 3. --- src/allmydata/test/no_network.py | 3 ++- src/allmydata/test/web/test_grid.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 9a2713830..e1f04b864 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -24,6 +24,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import unicode +from six import ensure_text import os from base64 import b32encode @@ -614,7 +615,7 @@ class GridTestMixin(object): method="GET", clientnum=0, **kwargs): # if return_response=True, this fires with (data, statuscode, # respheaders) instead of just data. - url = self.client_baseurls[clientnum] + urlpath + url = self.client_baseurls[clientnum] + ensure_text(urlpath) response = yield treq.request(method, url, persistent=False, allow_redirects=followRedirect, diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 04c3edbac..652d29953 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1,5 +1,7 @@ from __future__ import print_function +from past.builtins import unicode + import os.path, re from urllib.parse import quote as url_quote import json @@ -48,7 +50,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def CHECK(self, ign, which, args, clientnum=0): fileurl = self.fileurls[which] url = fileurl + "?" + args - return self.GET(url, method="POST", clientnum=clientnum) + return self.GET(url, method="POST", clientnum=clientnum).addCallback(unicode, "utf-8") def test_filecheck(self): self.basedir = "web/Grid/filecheck" @@ -1275,7 +1277,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d = c0.upload(upload.Data(DATA, convergence=b"")) def _stash_uri_and_create_dir(ur): self.uri = ur.get_uri() - self.url = "uri/"+self.uri + self.url = b"uri/"+self.uri u = uri.from_string_filenode(self.uri) self.si = u.get_storage_index() childnode = c0.create_node_from_uri(self.uri, None) @@ -1284,7 +1286,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def _stash_dir(node): self.dir_node = node self.dir_uri = node.get_uri() - self.dir_url = "uri/"+self.dir_uri + self.dir_url = b"uri/"+self.dir_uri d.addCallback(_stash_dir) d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True)) def _check_dir_html(body): From 98c71e51e1aaf66f8dac9c950fd9a5818b333325 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 10:04:27 -0500 Subject: [PATCH 04/27] More progress towards passing tests. --- src/allmydata/blacklist.py | 4 ++-- src/allmydata/test/common.py | 5 +++-- src/allmydata/test/web/test_grid.py | 11 +++++----- src/allmydata/web/directory.py | 34 ++++++++++++++--------------- src/allmydata/web/filenode.py | 2 +- src/allmydata/web/root.py | 1 + 6 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/allmydata/blacklist.py b/src/allmydata/blacklist.py index 89ee81a96..1ee507117 100644 --- a/src/allmydata/blacklist.py +++ b/src/allmydata/blacklist.py @@ -34,10 +34,10 @@ class Blacklist(object): try: if self.last_mtime is None or current_mtime > self.last_mtime: self.entries.clear() - with open(self.blacklist_fn, "r") as f: + with open(self.blacklist_fn, "rb") as f: for line in f: line = line.strip() - if not line or line.startswith("#"): + if not line or line.startswith(b"#"): continue si_s, reason = line.split(None, 1) si = base32.a2b(si_s) # must be valid base32 diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index b2fe8cac8..5bf90731d 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -825,11 +825,12 @@ class WebErrorMixin(object): code=None, substring=None, response_substring=None, callable=None, *args, **kwargs): # returns a Deferred with the response body - assert substring is None or isinstance(substring, str) + assert substring is None or isinstance(substring, bytes) + assert substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: - self.failUnlessEqual(f.value.status, str(code), which) + self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: code_string = str(f) self.failUnless(substring in code_string, diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 652d29953..6fcc2c06f 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1290,6 +1290,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_dir) d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True)) def _check_dir_html(body): + body = unicode(body, "utf-8") self.failUnlessIn(DIR_HTML_TAG, body) self.failUnlessIn("blacklisted.txt", body) d.addCallback(_check_dir_html) @@ -1301,14 +1302,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi f.write(" # this is a comment\n") f.write(" \n") f.write("\n") # also exercise blank lines - f.write("%s %s\n" % (base32.b2a(self.si), "off-limits to you")) + f.write("%s off-limits to you\n" % (unicode(base32.b2a(self.si), "ascii"),)) f.close() # clients should be checking the blacklist each time, so we don't # need to restart the client d.addCallback(_blacklist) d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri", - 403, "Forbidden", - "Access Prohibited: off-limits", + 403, b"Forbidden", + b"Access Prohibited: off-limits", self.GET, self.url)) # We should still be able to list the parent directory, in HTML... @@ -1376,8 +1377,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda body: self.failUnlessEqual(DATA, body)) def _block_dir(ign): - f = open(fn, "w") - f.write("%s %s\n" % (self.dir_si_b32, "dir-off-limits to you")) + f = open(fn, "wb") + f.write(b"%s %s\n" % (self.dir_si_b32, b"dir-off-limits to you")) f.close() self.g.clients[0].blacklist.last_mtime -= 2.0 d.addCallback(_block_dir) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index f83defd6a..31afd4187 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -1,6 +1,6 @@ import json -import urllib +from urllib.parse import quote as url_quote from datetime import timedelta from zope.interface import implementer @@ -109,7 +109,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # or no further children) renders "this" page. We also need # to reject "/uri/URI:DIR2:..//", so we look at postpath. name = name.decode('utf8') - if not name and req.postpath != ['']: + if not name and req.postpath != [b'']: return self # Rejecting URIs that contain empty path pieces (for example: @@ -135,7 +135,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name nonterminal = not terminal #len(req.postpath) > 0 - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() if isinstance(node_or_failure, Failure): f = node_or_failure f.trap(NoSuchChildError) @@ -217,7 +217,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): @render_exception def render_GET(self, req): # This is where all of the directory-related ?t=* code goes. - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() # t=info contains variable ophandles, t=rename-form contains the name # of the child being renamed. Neither is allowed an ETag. @@ -255,7 +255,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): @render_exception def render_PUT(self, req): - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() replace = parse_replace_arg(get_arg(req, "replace", "true")) if t == "mkdir": @@ -275,7 +275,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): @render_exception def render_POST(self, req): - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() if t == "mkdir": d = self._POST_mkdir(req) @@ -732,7 +732,7 @@ class DirectoryAsHTML(Element): return "" rocap = self.node.get_readonly_uri() root = get_root(req) - uri_link = "%s/uri/%s/" % (root, urllib.quote(rocap)) + uri_link = "%s/uri/%s/" % (root, url_quote(rocap)) return tag(tags.a("Read-Only Version", href=uri_link)) @renderer @@ -754,10 +754,10 @@ class DirectoryAsHTML(Element): called by the 'children' renderer) """ name = name.encode("utf-8") - nameurl = urllib.quote(name, safe="") # encode any slashes too + nameurl = url_quote(name, safe="") # encode any slashes too root = get_root(req) - here = "{}/uri/{}/".format(root, urllib.quote(self.node.get_uri())) + here = "{}/uri/{}/".format(root, url_quote(self.node.get_uri())) if self.node.is_unknown() or self.node.is_readonly(): unlink = "-" rename = "-" @@ -814,7 +814,7 @@ class DirectoryAsHTML(Element): assert IFilesystemNode.providedBy(target), target target_uri = target.get_uri() or "" - quoted_uri = urllib.quote(target_uri, safe="") # escape slashes too + quoted_uri = url_quote(target_uri, safe="") # escape slashes too if IMutableFileNode.providedBy(target): # to prevent javascript in displayed .html files from stealing a @@ -835,7 +835,7 @@ class DirectoryAsHTML(Element): elif IDirectoryNode.providedBy(target): # directory - uri_link = "%s/uri/%s/" % (root, urllib.quote(target_uri)) + uri_link = "%s/uri/%s/" % (root, url_quote(target_uri)) slots["filename"] = tags.a(name, href=uri_link) if not target.is_mutable(): dirtype = "DIR-IMM" @@ -871,7 +871,7 @@ class DirectoryAsHTML(Element): slots["size"] = "-" # use a directory-relative info link, so we can extract both the # writecap and the readcap - info_link = "%s?t=info" % urllib.quote(name) + info_link = "%s?t=info" % url_quote(name) if info_link: slots["info"] = tags.a("More Info", href=info_link) @@ -888,7 +888,7 @@ class DirectoryAsHTML(Element): # because action="." doesn't get us back to the dir page (but # instead /uri itself) root = get_root(req) - here = "{}/uri/{}/".format(root, urllib.quote(self.node.get_uri())) + here = "{}/uri/{}/".format(root, url_quote(self.node.get_uri())) if self.node.is_readonly(): return tags.div("No upload forms: directory is read-only") @@ -1166,13 +1166,13 @@ def _cap_to_link(root, path, cap): if isinstance(cap_obj, (CHKFileURI, WriteableSSKFileURI, ReadonlySSKFileURI)): uri_link = root_url.child( u"file", - u"{}".format(urllib.quote(cap)), - u"{}".format(urllib.quote(path[-1])), + u"{}".format(url_quote(cap)), + u"{}".format(url_quote(path[-1])), ) else: uri_link = root_url.child( u"uri", - u"{}".format(urllib.quote(cap, safe="")), + u"{}".format(url_quote(cap, safe="")), ) return tags.a(cap, href=uri_link.to_text()) else: @@ -1464,7 +1464,7 @@ class UnknownNodeHandler(Resource, object): @render_exception def render_GET(self, req): - t = get_arg(req, "t", "").strip() + t = get_arg(req, b"t", b"").strip() if t == "info": return MoreInfo(self.node) if t == "json": diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index b58cc71f9..a69690b41 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,4 +1,4 @@ -from past.builtins import unicode +from past.builtins import unicode, long import json diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 5d45d41b3..fa0a1c381 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -159,6 +159,7 @@ class URIHandler(resource.Resource, object): node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) except (TypeError, AssertionError): + raise raise WebError( "'{}' is not a valid file- or directory- cap".format(name) ) From 15c7af8e72a59f4702dff0470514369d02fed9d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 10:21:39 -0500 Subject: [PATCH 05/27] Another passing test. --- src/allmydata/test/common.py | 12 ++++++++---- src/allmydata/test/web/test_grid.py | 29 +++++++++++++++++------------ src/allmydata/web/directory.py | 8 ++++---- src/allmydata/web/filenode.py | 4 ++-- src/allmydata/web/root.py | 3 +-- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 5bf90731d..f1dbf651d 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -11,7 +11,7 @@ __all__ = [ "skipIf", ] -from past.builtins import chr as byteschr +from past.builtins import chr as byteschr, unicode import os, random, struct import six @@ -825,14 +825,18 @@ class WebErrorMixin(object): code=None, substring=None, response_substring=None, callable=None, *args, **kwargs): # returns a Deferred with the response body - assert substring is None or isinstance(substring, bytes) - assert substring is None or isinstance(response_substring, bytes) + if isinstance(substring, bytes): + substring = unicode(substring, "ascii") + if isinstance(response_substring, unicode): + response_substring = response_substring.encode("ascii") + assert substring is None or isinstance(substring, unicode) + assert response_substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: - code_string = str(f) + code_string = unicode(f) self.failUnless(substring in code_string, "%s: substring '%s' not in '%s'" % (which, substring, code_string)) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 6fcc2c06f..c78e0261f 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -52,6 +52,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi url = fileurl + "?" + args return self.GET(url, method="POST", clientnum=clientnum).addCallback(unicode, "utf-8") + def GET_string(self, *args, **kwargs): + """Send an HTTP request, but convert result to Unicode string.""" + d = GridTestMixin.GET(self, *args, **kwargs) + d.addCallback(unicode, "utf-8") + return d + def test_filecheck(self): self.basedir = "web/Grid/filecheck" self.set_up_grid() @@ -1288,9 +1294,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.dir_uri = node.get_uri() self.dir_url = b"uri/"+self.dir_uri d.addCallback(_stash_dir) - d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True)) + d.addCallback(lambda ign: self.GET_string(self.dir_url, followRedirect=True)) def _check_dir_html(body): - body = unicode(body, "utf-8") self.failUnlessIn(DIR_HTML_TAG, body) self.failUnlessIn("blacklisted.txt", body) d.addCallback(_check_dir_html) @@ -1308,19 +1313,19 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # need to restart the client d.addCallback(_blacklist) d.addCallback(lambda ign: self.shouldHTTPError("get_from_blacklisted_uri", - 403, b"Forbidden", - b"Access Prohibited: off-limits", + 403, "Forbidden", + "Access Prohibited: off-limits", self.GET, self.url)) # We should still be able to list the parent directory, in HTML... - d.addCallback(lambda ign: self.GET(self.dir_url, followRedirect=True)) + d.addCallback(lambda ign: self.GET_string(self.dir_url, followRedirect=True)) def _check_dir_html2(body): self.failUnlessIn(DIR_HTML_TAG, body) self.failUnlessIn("blacklisted.txt", body) d.addCallback(_check_dir_html2) # ... and in JSON (used by CLI). - d.addCallback(lambda ign: self.GET(self.dir_url+"?t=json", followRedirect=True)) + d.addCallback(lambda ign: self.GET(self.dir_url+b"?t=json", followRedirect=True)) def _check_dir_json(res): data = json.loads(res) self.failUnless(isinstance(data, list), data) @@ -1359,14 +1364,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_add_dir) def _get_dircap(dn): self.dir_si_b32 = base32.b2a(dn.get_storage_index()) - self.dir_url_base = "uri/"+dn.get_write_uri() - self.dir_url_json1 = "uri/"+dn.get_write_uri()+"?t=json" - self.dir_url_json2 = "uri/"+dn.get_write_uri()+"?t=json" - self.dir_url_json_ro = "uri/"+dn.get_readonly_uri()+"?t=json" - self.child_url = "uri/"+dn.get_readonly_uri()+"/child" + self.dir_url_base = b"uri/"+dn.get_write_uri() + self.dir_url_json1 = b"uri/"+dn.get_write_uri()+b"?t=json" + self.dir_url_json2 = b"uri/"+dn.get_write_uri()+b"?t=json" + self.dir_url_json_ro = b"uri/"+dn.get_readonly_uri()+b"?t=json" + self.child_url = b"uri/"+dn.get_readonly_uri()+b"/child" d.addCallback(_get_dircap) d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True)) - d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, body)) + d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, unicode(body, "utf-8"))) d.addCallback(lambda ign: self.GET(self.dir_url_json1)) d.addCallback(lambda res: json.loads(res)) # just check it decodes d.addCallback(lambda ign: self.GET(self.dir_url_json2)) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 31afd4187..d66f664c9 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -1,5 +1,5 @@ +from past.builtins import unicode -import json from urllib.parse import quote as url_quote from datetime import timedelta @@ -20,7 +20,7 @@ from twisted.web.template import ( from hyperlink import URL from twisted.python.filepath import FilePath -from allmydata.util import base32 +from allmydata.util import base32, jsonbytes as json from allmydata.util.encodingutil import ( to_bytes, quote_output, @@ -217,7 +217,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): @render_exception def render_GET(self, req): # This is where all of the directory-related ?t=* code goes. - t = get_arg(req, b"t", b"").strip() + t = unicode(get_arg(req, b"t", b"").strip(), "ascii") # t=info contains variable ophandles, t=rename-form contains the name # of the child being renamed. Neither is allowed an ETag. @@ -1005,7 +1005,7 @@ def _directory_json_metadata(req, dirnode): d = dirnode.list() def _got(children): kids = {} - for name, (childnode, metadata) in children.iteritems(): + for name, (childnode, metadata) in children.items(): assert IFilesystemNode.providedBy(childnode), childnode rw_uri = childnode.get_write_uri() ro_uri = childnode.get_readonly_uri() diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index a69690b41..5bd575631 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,7 +1,5 @@ from past.builtins import unicode, long -import json - from twisted.web import http, static from twisted.internet import defer from twisted.web.resource import ( @@ -42,6 +40,8 @@ from allmydata.web.check_results import ( LiteralCheckResultsRenderer, ) from allmydata.web.info import MoreInfo +from allmydata.util import jsonbytes as json + class ReplaceMeMixin(object): def replace_me_with_a_child(self, req, client, replace): diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index fa0a1c381..969f22a1b 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,6 +1,5 @@ import os import time -import json import urllib from hyperlink import DecodedURL, URL @@ -21,7 +20,7 @@ from twisted.web.template import ( ) import allmydata # to display import path -from allmydata.util import log +from allmydata.util import log, jsonbytes as json from allmydata.interfaces import IFileNode from allmydata.web import ( filenode, From d8197d95541b34fd3b8c62583160c547e8f4d733 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 10:52:31 -0500 Subject: [PATCH 06/27] Another passing test. --- src/allmydata/test/web/test_grid.py | 4 ++-- src/allmydata/web/directory.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index c78e0261f..1bcf528cc 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1004,12 +1004,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_uri, "one") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", + upload.Data(b"literal", convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: - c0.create_mutable_file(publish.MutableData("mutable"))) + c0.create_mutable_file(publish.MutableData(b"mutable"))) d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn)) d.addCallback(_stash_uri, "mutable") diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index d66f664c9..2ae5eac17 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -275,7 +275,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): @render_exception def render_POST(self, req): - t = get_arg(req, b"t", b"").strip() + t = unicode(get_arg(req, b"t", b"").strip(), "ascii") if t == "mkdir": d = self._POST_mkdir(req) @@ -1441,7 +1441,7 @@ class DeepCheckStreamer(dirnode.DeepStats): def write_line(self, data): j = json.dumps(data, ensure_ascii=True) assert "\n" not in j - self.req.write(j+"\n") + self.req.write(j.encode("utf-8")+b"\n") def finish(self): stats = dirnode.DeepStats.get_results(self) @@ -1450,8 +1450,8 @@ class DeepCheckStreamer(dirnode.DeepStats): } j = json.dumps(d, ensure_ascii=True) assert "\n" not in j - self.req.write(j+"\n") - return "" + self.req.write(j.encode("utf-8")+b"\n") + return b"" class UnknownNodeHandler(Resource, object): From a2f042845d6f8e0ca37c12bbf0e4aa7f85a81a2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 10:58:09 -0500 Subject: [PATCH 07/27] Another passing test. --- src/allmydata/test/web/test_grid.py | 6 +++--- src/allmydata/web/directory.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 1bcf528cc..cadd9e224 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -81,7 +81,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi c0.create_mutable_file(publish.MutableData(DATA+b"3"))) d.addCallback(_stash_mutable_uri, "corrupt") d.addCallback(lambda ign: - c0.upload(upload.Data("literal", convergence=b""))) + c0.upload(upload.Data(b"literal", convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: c0.create_immutable_dirnode({})) d.addCallback(_stash_mutable_uri, "smalldir") @@ -618,7 +618,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_uri, "good") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", + upload.Data(b"literal", convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: @@ -791,7 +791,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_stash_uri, "good") d.addCallback(lambda ign: self.rootnode.add_file(u"small", - upload.Data("literal", + upload.Data(b"literal", convergence=b""))) d.addCallback(_stash_uri, "small") d.addCallback(lambda ign: diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 2ae5eac17..56ad9a85c 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -225,7 +225,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES: si = self.node.get_storage_index() if si and req.setETag('DIR:%s-%s' % (base32.b2a(si), t or "")): - return "" + return b"" if not t: # render the directory as HTML @@ -1363,7 +1363,7 @@ class ManifestStreamer(dirnode.DeepStats): j = json.dumps(d, ensure_ascii=True) assert "\n" not in j - self.req.write(j+"\n") + self.req.write(j.encode("utf-8")+b"\n") def finish(self): stats = dirnode.DeepStats.get_results(self) @@ -1372,8 +1372,8 @@ class ManifestStreamer(dirnode.DeepStats): } j = json.dumps(d, ensure_ascii=True) assert "\n" not in j - self.req.write(j+"\n") - return "" + self.req.write(j.encode("utf-8")+b"\n") + return b"" @implementer(IPushProducer) class DeepCheckStreamer(dirnode.DeepStats): From 2737229895bb55437cdd7140fd3e23a0eedf54fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 11:12:52 -0500 Subject: [PATCH 08/27] Another passing test. --- src/allmydata/test/common_util.py | 14 +++++--------- src/allmydata/test/web/test_grid.py | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 1ad0f8242..168714492 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,7 +1,8 @@ from __future__ import print_function -from future.utils import PY2, native_str +from future.utils import PY2, native_str, bchr, binary_type from future.builtins import str as future_str +from past.builtins import unicode import os import time @@ -20,9 +21,6 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding -# Imported for backwards compatibility: -from future.utils import bord, bchr, binary_type -from past.builtins import unicode def skip_if_cannot_represent_filename(u): @@ -183,13 +181,11 @@ def insecurerandstr(n): return b''.join(map(bchr, map(randrange, [0]*n, [256]*n))) def flip_bit(good, which): - # TODO Probs need to update with bchr/bord as with flip_one_bit, below. - # flip the low-order bit of good[which] if which == -1: - pieces = good[:which], good[-1:], "" + pieces = good[:which], good[-1:], b"" else: pieces = good[:which], good[which:which+1], good[which+1:] - return pieces[0] + chr(ord(pieces[1]) ^ 0x01) + pieces[2] + return pieces[0] + bchr(ord(pieces[1]) ^ 0x01) + pieces[2] def flip_one_bit(s, offset=0, size=None): """ flip one random bit of the string s, in a byte greater than or equal to offset and less @@ -198,7 +194,7 @@ def flip_one_bit(s, offset=0, size=None): if size is None: size=len(s)-offset i = randrange(offset, offset+size) - result = s[:i] + bchr(bord(s[i])^(0x01<Return to file', res) @@ -1101,6 +1103,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 410, "Gone", "NoSharesError", self.GET, self.fileurls["0shares"])) def _check_zero_shares(body): + body = unicode(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) exp = ("NoSharesError: no shares could be found. " @@ -1118,6 +1121,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 410, "Gone", "NotEnoughSharesError", self.GET, self.fileurls["1share"])) def _check_one_share(body): + body = unicode(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) msgbase = ("NotEnoughSharesError: This indicates that some " @@ -1142,10 +1146,11 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 404, "Not Found", None, self.GET, self.fileurls["imaginary"])) def _missing_child(body): + body = unicode(body, "utf-8") self.failUnlessIn("No such child: imaginary", body) d.addCallback(_missing_child) - d.addCallback(lambda ignored: self.GET(self.fileurls["dir-0share"])) + d.addCallback(lambda ignored: self.GET_string(self.fileurls["dir-0share"])) def _check_0shares_dir_html(body): self.failUnlessIn(DIR_HTML_TAG, body) # we should see the regular page, but without the child table or @@ -1164,7 +1169,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("No upload forms: directory is unreadable", body) d.addCallback(_check_0shares_dir_html) - d.addCallback(lambda ignored: self.GET(self.fileurls["dir-1share"])) + d.addCallback(lambda ignored: self.GET_string(self.fileurls["dir-1share"])) def _check_1shares_dir_html(body): # at some point, we'll split UnrecoverableFileError into 0-shares # and some-shares like we did for immutable files (since there @@ -1191,6 +1196,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["dir-0share-json"])) def _check_unrecoverable_file(body): + body = unicode(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) exp = ("UnrecoverableFileError: the directory (or mutable file) " @@ -1218,7 +1224,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # attach a webapi child that throws a random error, to test how it # gets rendered. w = c0.getServiceNamed("webish") - w.root.putChild("ERRORBOOM", ErrorBoom()) + w.root.putChild(b"ERRORBOOM", ErrorBoom()) # "Accept: */*" : should get a text/html stack trace # "Accept: text/plain" : should get a text/plain stack trace @@ -1231,6 +1237,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": "*/*"})) def _internal_error_html1(body): + body = unicode(body, "utf-8") self.failUnlessIn("", "expected HTML, not '%s'" % body) d.addCallback(_internal_error_html1) @@ -1240,6 +1247,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": "text/plain"})) def _internal_error_text2(body): + body = unicode(body, "utf-8") self.failIfIn("", body) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text2) @@ -1251,6 +1259,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": CLI_accepts})) def _internal_error_text3(body): + body = unicode(body, "utf-8") self.failIfIn("", body) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text3) @@ -1260,7 +1269,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 500, "Internal Server Error", None, self.GET, "ERRORBOOM")) def _internal_error_html4(body): - self.failUnlessIn("", body) + self.failUnlessIn(b"", body) d.addCallback(_internal_error_html4) def _flush_errors(res): From c25dd5776888b0d3ec9e0ecf24d16d180376890c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 21 Dec 2020 13:12:01 -0500 Subject: [PATCH 09/27] Make sure we can handle bytes, plus a couple other fixes. --- src/allmydata/dirnode.py | 3 +-- src/allmydata/scripts/debug.py | 6 ++++-- src/allmydata/test/web/test_grid.py | 2 -- src/allmydata/web/introweb.py | 3 +-- src/allmydata/web/status.py | 3 +-- src/allmydata/web/storage.py | 4 ++-- src/allmydata/webish.py | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index fd6a9cc8c..e8b80b9ad 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -18,7 +18,6 @@ import time from zope.interface import implementer from twisted.internet import defer from foolscap.api import fireEventually -import json from allmydata.crypto import aes from allmydata.deep_stats import DeepStats @@ -31,7 +30,7 @@ from allmydata.interfaces import IFilesystemNode, IDirectoryNode, IFileNode, \ from allmydata.check_results import DeepCheckResults, \ DeepCheckAndRepairResults from allmydata.monitor import Monitor -from allmydata.util import hashutil, base32, log +from allmydata.util import hashutil, base32, log, jsonbytes as json from allmydata.util.encodingutil import quote_output, normalize from allmydata.util.assertutil import precondition from allmydata.util.netstring import netstring, split_netstring diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fd3f2b87c..451b1d661 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,7 @@ from __future__ import print_function +from future.utils import bchr + # do not import any allmydata modules at this level. Do that from inside # individual functions instead. import struct, time, os, sys @@ -905,7 +907,7 @@ def corrupt_share(options): f = open(fn, "rb+") f.seek(offset) d = f.read(1) - d = chr(ord(d) ^ 0x01) + d = bchr(ord(d) ^ 0x01) f.seek(offset) f.write(d) f.close() @@ -920,7 +922,7 @@ def corrupt_share(options): f.seek(m.DATA_OFFSET) data = f.read(2000) # make sure this slot contains an SMDF share - assert data[0] == b"\x00", "non-SDMF mutable shares not supported" + assert data[0:1] == b"\x00", "non-SDMF mutable shares not supported" f.close() (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize, diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 0b58e2956..ecdc454b6 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -109,7 +109,6 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(self.CHECK, "good", "t=check") def _got_html_good(res): - res = unicode(res, "utf-8") self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) soup = BeautifulSoup(res, 'html5lib') @@ -118,7 +117,6 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_got_html_good) d.addCallback(self.CHECK, "good", "t=check&return_to=somewhere") def _got_html_good_return_to(res): - res = unicode(res, "utf-8") self.failUnlessIn("Healthy", res) self.failIfIn("Not Healthy", res) self.failUnlessIn('Return to file', res) diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 380b6efd4..6ec558e82 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -5,8 +5,7 @@ from twisted.web.template import Element, XMLFile, renderElement, renderer from twisted.python.filepath import FilePath from twisted.web import static import allmydata -import json -from allmydata.util import idlib +from allmydata.util import idlib, jsonbytes as json from allmydata.web.common import ( render_time, MultiFormatResource, diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3284dfda6..2002b2fdf 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -3,7 +3,6 @@ from past.builtins import long, unicode import pprint import itertools import hashlib -import json from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.web.resource import Resource @@ -14,7 +13,7 @@ from twisted.web.template import ( renderElement, tags, ) -from allmydata.util import base32, idlib +from allmydata.util import base32, idlib, jsonbytes as json from allmydata.web.common import ( abbreviate_time, abbreviate_rate, diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 51624a409..82c789d9b 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,6 +1,6 @@ from future.utils import PY2 -import time, json +import time from twisted.python.filepath import FilePath from twisted.web.template import ( Element, @@ -14,7 +14,7 @@ from allmydata.web.common_py3 import ( MultiFormatResource ) from allmydata.util.abbreviate import abbreviate_space -from allmydata.util import time_format, idlib +from allmydata.util import time_format, idlib, jsonbytes as json def remove_prefix(s, prefix): diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index a7f44514f..3bfd073a4 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -110,7 +110,7 @@ def _get_client_ip(request): def _logFormatter(logDateTime, request): - print("REQUEST: {}".format(request.uri)) + print("REQUEST: {} {}".format(request.method, request.uri)) # we build up a log string that hides most of the cap, to preserve # user privacy. We retain the query args so we can identify things # like t=json. Then we send it to the flog. We make no attempt to From 0534979e617ba57aa011eaf57b0c076c67dd538c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 11:03:23 -0500 Subject: [PATCH 10/27] Another passing test on Python 3. --- src/allmydata/web/common_py3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 05fc2b50b..9d4435706 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -70,6 +70,8 @@ class MultiFormatResource(resource.Resource, object): :return: The result of the selected renderer. """ t = get_arg(req, self.formatArgument, self.formatDefault) + if isinstance(t, bytes): + t = unicode(t, "ascii") renderer = self._get_renderer(t) return renderer(req) From 74c08883f5c8eaaa171a396bb89db04b0c4bc2e8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 11:36:52 -0500 Subject: [PATCH 11/27] Another passing test on Python 3. --- src/allmydata/test/web/test_grid.py | 22 ++++++++++++---------- src/allmydata/web/directory.py | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index ecdc454b6..f140bc69f 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -361,18 +361,19 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # make sure directory listing tolerates unknown nodes d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_directory_html(res, expected_type_suffix): - pattern = re.compile(r'\?%s[ \t\n\r]*' - '%s' % (expected_type_suffix, str(name)), + pattern = re.compile(br'\?%s[ \t\n\r]*' + b'%s' % ( + expected_type_suffix, name.encode("ascii")), re.DOTALL) self.failUnless(re.search(pattern, res), res) # find the More Info link for name, should be relative - mo = re.search(r'More Info', res) + mo = re.search(br'More Info', res) info_url = mo.group(1) - self.failUnlessReallyEqual(info_url, "%s?t=info" % (str(name),)) + self.failUnlessReallyEqual(info_url, b"%s?t=info" % (name.encode("ascii"),)) if immutable: - d.addCallback(_check_directory_html, "-IMM") + d.addCallback(_check_directory_html, b"-IMM") else: - d.addCallback(_check_directory_html, "") + d.addCallback(_check_directory_html, b"") d.addCallback(lambda ign: self.GET(self.rooturl+"?t=json")) def _check_directory_json(res, expect_rw_uri): @@ -392,7 +393,6 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(_check_directory_json, expect_rw_uri=not immutable) def _check_info(res, expect_rw_uri, expect_ro_uri): - self.failUnlessIn("Object Type: unknown", res) if expect_rw_uri: self.failUnlessIn(unknown_rwcap, res) if expect_ro_uri: @@ -402,6 +402,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn(unknown_rocap, res) else: self.failIfIn(unknown_rocap, res) + res = unicode(res, "utf-8") + self.failUnlessIn("Object Type: unknown", res) self.failIfIn("Raw data as", res) self.failIfIn("Directory writecap", res) self.failIfIn("Checker Operations", res) @@ -413,7 +415,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(expected_info_url)) d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=False) - d.addCallback(lambda ign: self.GET("%s/%s?t=info" % (self.rooturl, str(name)))) + d.addCallback(lambda ign: self.GET("%s/%s?t=info" % (self.rooturl, name))) d.addCallback(_check_info, expect_rw_uri=False, expect_ro_uri=True) def _check_json(res, expect_rw_uri): @@ -445,9 +447,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi # or not future_node was immutable. d.addCallback(lambda ign: self.GET(self.rourl)) if immutable: - d.addCallback(_check_directory_html, "-IMM") + d.addCallback(_check_directory_html, b"-IMM") else: - d.addCallback(_check_directory_html, "-RO") + d.addCallback(_check_directory_html, b"-RO") d.addCallback(lambda ign: self.GET(self.rourl+"?t=json")) d.addCallback(_check_directory_json, expect_rw_uri=False) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 56ad9a85c..981c8ef56 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -1464,7 +1464,7 @@ class UnknownNodeHandler(Resource, object): @render_exception def render_GET(self, req): - t = get_arg(req, b"t", b"").strip() + t = unicode(get_arg(req, "t", "").strip(), "ascii") if t == "info": return MoreInfo(self.node) if t == "json": From baa2cff29ceb50c821aa63b6aa8926dd5f1f3d77 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 13:14:18 -0500 Subject: [PATCH 12/27] Unbreak Python 2. --- src/allmydata/test/web/test_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index f140bc69f..c4d212133 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1112,7 +1112,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi "severe corruption. You should perform a filecheck on " "this object to learn more. The full error message is: " "no shares (need 3). Last failure: None") - self.failUnlessReallyEqual(exp, body) + self.assertEqual(exp, body) d.addCallback(_check_zero_shares) From 8881728ca52c7fb1bfdf8d74fd8a9a939c308cb9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 13:17:07 -0500 Subject: [PATCH 13/27] Another passing test on Python 3. --- src/allmydata/test/web/test_grid.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index c4d212133..67496eb1e 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -473,9 +473,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.uris = {} self.fileurls = {} - lonely_uri = "URI:LIT:n5xgk" # LIT for "one" - mut_write_uri = "URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" - mut_read_uri = "URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" + lonely_uri = b"URI:LIT:n5xgk" # LIT for "one" + mut_write_uri = b"URI:SSK:vfvcbdfbszyrsaxchgevhmmlii:euw4iw7bbnkrrwpzuburbhppuxhc3gwxv26f6imekhz7zyw2ojnq" + mut_read_uri = b"URI:SSK-RO:e3mdrzfwhoq42hy5ubcz6rp3o4:ybyibhnp3vvwuq2vaw2ckjmesgkklfs6ghxleztqidihjyofgw7q" # This method tests mainly dirnode, but we'd have to duplicate code in order to # test the dirnode and web layers separately. @@ -518,7 +518,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi rep = str(dn) self.failUnlessIn("RO-IMM", rep) cap = dn.get_cap() - self.failUnlessIn("CHK", cap.to_string()) + self.failUnlessIn(b"CHK", cap.to_string()) self.cap = cap self.rootnode = dn self.rooturl = "uri/" + url_quote(dn.get_uri()) @@ -537,7 +537,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi entry = entries[0] (name_utf8, ro_uri, rwcapdata, metadata_s), subpos = split_netstring(entry, 4) name = name_utf8.decode("utf-8") - self.failUnlessEqual(rwcapdata, "") + self.failUnlessEqual(rwcapdata, b"") self.failUnlessIn(name, kids) (expected_child, ign) = kids[name] self.failUnlessReallyEqual(ro_uri, expected_child.get_readonly_uri()) @@ -564,7 +564,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.GET(self.rooturl)) def _check_html(res): soup = BeautifulSoup(res, 'html5lib') - self.failIfIn("URI:SSK", res) + self.failIfIn(b"URI:SSK", res) found = False for td in soup.find_all(u"td"): if td.text != u"FILE": From cbf348f21b379799376dd861d3af69eed76e2a60 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 13:17:46 -0500 Subject: [PATCH 14/27] Get rid of debug print. --- src/allmydata/webish.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index 3bfd073a4..f32f56714 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -110,7 +110,6 @@ def _get_client_ip(request): def _logFormatter(logDateTime, request): - print("REQUEST: {} {}".format(request.method, request.uri)) # we build up a log string that hides most of the cap, to preserve # user privacy. We retain the query args so we can identify things # like t=json. Then we send it to the flog. We make no attempt to From 3ca17454c4a2f277dc1dc826b640007dee863560 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 13:18:07 -0500 Subject: [PATCH 15/27] News file. --- newsfragments/3566.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3566.minor diff --git a/newsfragments/3566.minor b/newsfragments/3566.minor new file mode 100644 index 000000000..e69de29bb From 1c7956bc1aad33fd3f3a666b7845ad6ea619a922 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 22 Dec 2020 13:19:59 -0500 Subject: [PATCH 16/27] Port to Python 3. --- src/allmydata/test/web/test_grid.py | 42 +++++++++++++++++------------ src/allmydata/util/_python3.py | 1 + 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 67496eb1e..7e5468143 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path, re from urllib.parse import quote as url_quote @@ -50,12 +58,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def CHECK(self, ign, which, args, clientnum=0): fileurl = self.fileurls[which] url = fileurl + "?" + args - return self.GET(url, method="POST", clientnum=clientnum).addCallback(unicode, "utf-8") + return self.GET(url, method="POST", clientnum=clientnum).addCallback(str, "utf-8") def GET_string(self, *args, **kwargs): """Send an HTTP request, but convert result to Unicode string.""" d = GridTestMixin.GET(self, *args, **kwargs) - d.addCallback(unicode, "utf-8") + d.addCallback(str, "utf-8") return d def test_filecheck(self): @@ -402,7 +410,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn(unknown_rocap, res) else: self.failIfIn(unknown_rocap, res) - res = unicode(res, "utf-8") + res = str(res, "utf-8") self.failUnlessIn("Object Type: unknown", res) self.failIfIn("Raw data as", res) self.failIfIn("Directory writecap", res) @@ -712,7 +720,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: self.delete_shares_numbered(self.uris["subdir"], - range(1, 10))) + list(range(1, 10)))) # root # root/good @@ -1072,7 +1080,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence=b""))) def _stash_bad(ur): self.fileurls["1share"] = "uri/" + url_quote(ur.get_uri()) - self.delete_shares_numbered(ur.get_uri(), range(1,10)) + self.delete_shares_numbered(ur.get_uri(), list(range(1,10))) u = uri.from_string(ur.get_uri()) u.key = testutil.flip_bit(u.key, 0) @@ -1084,14 +1092,14 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi u = n.get_uri() url = self.fileurls["dir-1share"] = "uri/" + url_quote(u) self.fileurls["dir-1share-json"] = url + "?t=json" - self.delete_shares_numbered(u, range(1,10)) + self.delete_shares_numbered(u, list(range(1,10))) d.addCallback(_mangle_dirnode_1share) d.addCallback(lambda ign: c0.create_dirnode()) def _mangle_dirnode_0share(n): u = n.get_uri() url = self.fileurls["dir-0share"] = "uri/" + url_quote(u) self.fileurls["dir-0share-json"] = url + "?t=json" - self.delete_shares_numbered(u, range(0,10)) + self.delete_shares_numbered(u, list(range(0,10))) d.addCallback(_mangle_dirnode_0share) # NotEnoughSharesError should be reported sensibly, with a @@ -1103,7 +1111,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 410, "Gone", "NoSharesError", self.GET, self.fileurls["0shares"])) def _check_zero_shares(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) exp = ("NoSharesError: no shares could be found. " @@ -1121,7 +1129,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 410, "Gone", "NotEnoughSharesError", self.GET, self.fileurls["1share"])) def _check_one_share(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) msgbase = ("NotEnoughSharesError: This indicates that some " @@ -1146,7 +1154,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi 404, "Not Found", None, self.GET, self.fileurls["imaginary"])) def _missing_child(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failUnlessIn("No such child: imaginary", body) d.addCallback(_missing_child) @@ -1196,7 +1204,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.fileurls["dir-0share-json"])) def _check_unrecoverable_file(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failIfIn("", body) body = " ".join(body.strip().split()) exp = ("UnrecoverableFileError: the directory (or mutable file) " @@ -1237,7 +1245,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": "*/*"})) def _internal_error_html1(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failUnlessIn("", "expected HTML, not '%s'" % body) d.addCallback(_internal_error_html1) @@ -1247,7 +1255,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": "text/plain"})) def _internal_error_text2(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failIfIn("", body) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text2) @@ -1259,7 +1267,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, "ERRORBOOM", headers={"accept": CLI_accepts})) def _internal_error_text3(body): - body = unicode(body, "utf-8") + body = str(body, "utf-8") self.failIfIn("", body) self.failUnless(body.startswith("Traceback "), body) d.addCallback(_internal_error_text3) @@ -1316,7 +1324,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi f.write(" # this is a comment\n") f.write(" \n") f.write("\n") # also exercise blank lines - f.write("%s off-limits to you\n" % (unicode(base32.b2a(self.si), "ascii"),)) + f.write("%s off-limits to you\n" % (str(base32.b2a(self.si), "ascii"),)) f.close() # clients should be checking the blacklist each time, so we don't # need to restart the client @@ -1380,7 +1388,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.child_url = b"uri/"+dn.get_readonly_uri()+b"/child" d.addCallback(_get_dircap) d.addCallback(lambda ign: self.GET(self.dir_url_base, followRedirect=True)) - d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, unicode(body, "utf-8"))) + d.addCallback(lambda body: self.failUnlessIn(DIR_HTML_TAG, str(body, "utf-8"))) d.addCallback(lambda ign: self.GET(self.dir_url_json1)) d.addCallback(lambda res: json.loads(res)) # just check it decodes d.addCallback(lambda ign: self.GET(self.dir_url_json2)) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index af771cd5a..a7b77001a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -180,6 +180,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_uri", "allmydata.test.test_util", "allmydata.test.web.test_common", + "allmydata.test.web.test_grid", "allmydata.test.web.test_util", "allmydata.test.web.test_status", ] From c5b403bd2f35717ee9cecf968696ee534295ff88 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Dec 2020 09:17:39 -0500 Subject: [PATCH 17/27] Make the class new style again on Python 2. --- src/allmydata/test/web/test_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 7e5468143..f85538296 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -48,7 +48,7 @@ DIR_HTML_TAG = '' class CompletelyUnhandledError(Exception): pass -class ErrorBoom(resource.Resource): +class ErrorBoom(resource.Resource, object): @render_exception def render(self, req): raise CompletelyUnhandledError("whoops") From f736dc6f7b1339b97594d8389617afa11c8bffea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Dec 2020 09:34:48 -0500 Subject: [PATCH 18/27] Fix some tests caused by unicode rendering. --- src/allmydata/test/web/test_web.py | 5 ++++- src/allmydata/web/common.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 1908afdeb..e975464d3 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -746,7 +746,10 @@ class MultiFormatResourceTests(TrialTestCase): "400 - Bad Format", response_body, ) self.assertIn( - "Unknown t value: 'foo'", response_body, + "Unknown t value:", response_body, + ) + self.assertIn( + "'foo'", response_body, ) diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 2832cc6a8..57118d1d4 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -1,5 +1,5 @@ from past.builtins import unicode -from six import ensure_text +from six import ensure_text, ensure_str import time import json @@ -112,7 +112,7 @@ def parse_replace_arg(replace): try: return boolean_of_arg(replace) except WebError: - raise WebError("invalid replace= argument: %r" % (replace,), http.BAD_REQUEST) + raise WebError("invalid replace= argument: %r" % (ensure_str(replace),), http.BAD_REQUEST) def get_format(req, default="CHK"): From 58cb757816bdf84f369d32d029da1d730c916bea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Dec 2020 09:42:42 -0500 Subject: [PATCH 19/27] Sometimes these values are more extended Unicode than ASCII. --- src/allmydata/web/common_py3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index b91f53a3a..5aa338e4d 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -29,9 +29,9 @@ def get_arg(req, argname, default=None, multiple=False): :return: Either bytes or tuple of bytes. """ if isinstance(argname, unicode): - argname = argname.encode("ascii") + argname = argname.encode("utf-8") if isinstance(default, unicode): - default = default.encode("ascii") + default = default.encode("utf-8") results = [] if argname in req.args: results.extend(req.args[argname]) From 3c8550b666c15937029184241e51a12b6a2e3739 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Dec 2020 09:54:35 -0500 Subject: [PATCH 20/27] Python 3 fix: direct indexing of bytes returns an int. --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 16e744902..74f15a464 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -194,7 +194,7 @@ def flip_one_bit(s, offset=0, size=None): if size is None: size=len(s)-offset i = randrange(offset, offset+size) - result = s[:i] + bchr(ord(s[i])^(0x01< Date: Wed, 23 Dec 2020 10:01:44 -0500 Subject: [PATCH 21/27] Fix tests on Python 3. --- src/allmydata/test/test_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 936d270ae..a7042468a 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -173,7 +173,7 @@ class WebResultsRendering(unittest.TestCase): return c def render_json(self, resource): - return self.successResultOf(render(resource, {"output": ["json"]})) + return self.successResultOf(render(resource, {b"output": [b"json"]})) def render_element(self, element, args=None): if args is None: @@ -186,7 +186,7 @@ class WebResultsRendering(unittest.TestCase): html = self.render_element(lcr) self.failUnlessIn(b"Literal files are always healthy", html) - html = self.render_element(lcr, args={"return_to": ["FOOURL"]}) + html = self.render_element(lcr, args={b"return_to": [b"FOOURL"]}) self.failUnlessIn(b"Literal files are always healthy", html) self.failUnlessIn(b'Return to file.', html) @@ -269,7 +269,7 @@ class WebResultsRendering(unittest.TestCase): self.failUnlessIn("File Check Results for SI=2k6avp", s) # abbreviated self.failUnlessIn("Not Recoverable! : rather dead", s) - html = self.render_element(w, args={"return_to": ["FOOURL"]}) + html = self.render_element(w, args={b"return_to": [b"FOOURL"]}) self.failUnlessIn(b'Return to file/directory.', html) From eb8837a4c8b0a22d607692248254f3220c1faf8b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 23 Dec 2020 10:09:37 -0500 Subject: [PATCH 22/27] More things that need to be bytes. --- src/allmydata/test/test_storage_web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py index ca0cd85fc..b3f5fac98 100644 --- a/src/allmydata/test/test_storage_web.py +++ b/src/allmydata/test/test_storage_web.py @@ -70,7 +70,7 @@ def renderJSON(resource): """ Render a JSON from the given resource. """ - return render(resource, {"t": ["json"]}) + return render(resource, {b"t": [b"json"]}) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): From a46a7dc7f8049f32dfd5a19dc27779a22f0953a1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Jan 2021 14:23:12 -0500 Subject: [PATCH 23/27] Log, don't raise. --- src/allmydata/web/root.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 969f22a1b..fdc72ab71 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -157,8 +157,9 @@ class URIHandler(resource.Resource, object): try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) - except (TypeError, AssertionError): - raise + except (TypeError, AssertionError) as e: + log.msg(format="Failed to parse cap, perhaps due to bug: %(e)s", + e=e, level=log.WEIRD) raise WebError( "'{}' is not a valid file- or directory- cap".format(name) ) From d7db34f27a0fc3c11f5169d09168044963185d42 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Jan 2021 14:33:06 -0500 Subject: [PATCH 24/27] Add explanation for if statement. --- src/allmydata/web/common_py3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 5aa338e4d..3e9eb8379 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -70,6 +70,7 @@ class MultiFormatResource(resource.Resource, object): :return: The result of the selected renderer. """ t = get_arg(req, self.formatArgument, self.formatDefault) + # It's either bytes or None. if isinstance(t, bytes): t = unicode(t, "ascii") renderer = self._get_renderer(t) From 6f0838e2e9a2c8f9b7366c82d4b2e46d4f00d671 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Jan 2021 14:34:18 -0500 Subject: [PATCH 25/27] Docstring. --- src/allmydata/test/common_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 74f15a464..2a70cff3a 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -181,6 +181,7 @@ def insecurerandstr(n): return b''.join(map(bchr, map(randrange, [0]*n, [256]*n))) def flip_bit(good, which): + """Flip the low-order bit of good[which].""" if which == -1: pieces = good[:which], good[-1:], b"" else: From 961ad123cc57b46580a1f77dbc44da67a5440588 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Jan 2021 14:35:33 -0500 Subject: [PATCH 26/27] Better name. --- src/allmydata/test/web/test_grid.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index f85538296..135957ed4 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -60,7 +60,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi url = fileurl + "?" + args return self.GET(url, method="POST", clientnum=clientnum).addCallback(str, "utf-8") - def GET_string(self, *args, **kwargs): + def GET_unicode(self, *args, **kwargs): """Send an HTTP request, but convert result to Unicode string.""" d = GridTestMixin.GET(self, *args, **kwargs) d.addCallback(str, "utf-8") @@ -1158,7 +1158,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("No such child: imaginary", body) d.addCallback(_missing_child) - d.addCallback(lambda ignored: self.GET_string(self.fileurls["dir-0share"])) + d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-0share"])) def _check_0shares_dir_html(body): self.failUnlessIn(DIR_HTML_TAG, body) # we should see the regular page, but without the child table or @@ -1177,7 +1177,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.failUnlessIn("No upload forms: directory is unreadable", body) d.addCallback(_check_0shares_dir_html) - d.addCallback(lambda ignored: self.GET_string(self.fileurls["dir-1share"])) + d.addCallback(lambda ignored: self.GET_unicode(self.fileurls["dir-1share"])) def _check_1shares_dir_html(body): # at some point, we'll split UnrecoverableFileError into 0-shares # and some-shares like we did for immutable files (since there @@ -1311,7 +1311,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.dir_uri = node.get_uri() self.dir_url = b"uri/"+self.dir_uri d.addCallback(_stash_dir) - d.addCallback(lambda ign: self.GET_string(self.dir_url, followRedirect=True)) + d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html(body): self.failUnlessIn(DIR_HTML_TAG, body) self.failUnlessIn("blacklisted.txt", body) @@ -1335,7 +1335,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi self.GET, self.url)) # We should still be able to list the parent directory, in HTML... - d.addCallback(lambda ign: self.GET_string(self.dir_url, followRedirect=True)) + d.addCallback(lambda ign: self.GET_unicode(self.dir_url, followRedirect=True)) def _check_dir_html2(body): self.failUnlessIn(DIR_HTML_TAG, body) self.failUnlessIn("blacklisted.txt", body) From bc19ccc77a358f968547f3d7500d117a925be1c0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Jan 2021 14:36:02 -0500 Subject: [PATCH 27/27] Use method that already does this. --- src/allmydata/test/web/test_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_grid.py b/src/allmydata/test/web/test_grid.py index 135957ed4..ef2718df4 100644 --- a/src/allmydata/test/web/test_grid.py +++ b/src/allmydata/test/web/test_grid.py @@ -58,7 +58,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi def CHECK(self, ign, which, args, clientnum=0): fileurl = self.fileurls[which] url = fileurl + "?" + args - return self.GET(url, method="POST", clientnum=clientnum).addCallback(str, "utf-8") + return self.GET_unicode(url, method="POST", clientnum=clientnum) def GET_unicode(self, *args, **kwargs): """Send an HTTP request, but convert result to Unicode string."""