From 196435784e35ebd2fd9774d4e46f1ce6c0c4ef8a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 Mar 2021 09:24:29 -0500 Subject: [PATCH 1/7] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/web/status.py | 42 ++++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 48c491190..74aa1cea8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -127,6 +127,7 @@ PORTED_MODULES = [ "allmydata.web.operations", "allmydata.web.private", "allmydata.web.root", + "allmydata.web.status", "allmydata.webish", ] diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 935c2c36c..e66966574 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1,4 +1,16 @@ -from past.builtins import long, unicode +""" +Ported to Python 3. +""" + +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long import itertools import hashlib @@ -60,7 +72,7 @@ class UploadResultsRendererMixin(Element): return "None" ul = tags.ul() for shnum, servers in sorted(sharemap.items()): - server_names = ', '.join([unicode(s.get_name(), "utf-8") for s in servers]) + server_names = ', '.join([str(s.get_name(), "utf-8") for s in servers]) ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) return ul d.addCallback(_render) @@ -76,7 +88,7 @@ class UploadResultsRendererMixin(Element): ul = tags.ul() for server, shnums in sorted(servermap.items(), key=id): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) - ul(tags.li("[%s] got share%s: %s" % (unicode(server.get_name(), "utf-8"), + ul(tags.li("[%s] got share%s: %s" % (str(server.get_name(), "utf-8"), plural(shnums), shares_s))) return ul d.addCallback(_render) @@ -231,7 +243,7 @@ class UploadStatusElement(UploadResultsRendererMixin): if si_s is None: si_s = "(None)" else: - si_s = unicode(si_s, "utf-8") + si_s = str(si_s, "utf-8") return tag(si_s) @renderer @@ -467,10 +479,10 @@ class DownloadStatusElement(Element): return "" return "+%.6fs" % t - def _rate_and_time(self, bytes, seconds): + def _rate_and_time(self, bytes_count, seconds): time_s = abbreviate_time(seconds) if seconds != 0: - rate = abbreviate_rate(1.0 * bytes / seconds) + rate = abbreviate_rate(bytes_count / seconds) return tags.span(time_s, title=rate) return tags.span(time_s) @@ -919,10 +931,10 @@ class RetrieveStatusElement(Element): if not per_server: return tag("") l = tags.ul() - for server in sorted(per_server.keys(), key=lambda s: s.get_name()): + for server in sorted(list(per_server.keys()), key=lambda s: s.get_name()): times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) - l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) + l(tags.li("[%s]: %s" % (str(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Fetch Response Times: ", l) @@ -961,7 +973,7 @@ class PublishStatusElement(Element): if si_s is None: si_s = "(None)" else: - si_s = unicode(si_s, "utf-8") + si_s = str(si_s, "utf-8") return tag(si_s) @renderer @@ -1000,7 +1012,7 @@ class PublishStatusElement(Element): sharemap = servermap.make_sharemap() for shnum in sorted(sharemap.keys()): l(tags.li("%d -> Placed on " % shnum, - ", ".join(["[%s]" % unicode(server.get_name(), "utf-8") + ", ".join(["[%s]" % str(server.get_name(), "utf-8") for server in sharemap[shnum]]))) return tag("Sharemap:", l) @@ -1079,10 +1091,10 @@ class PublishStatusElement(Element): if not per_server: return tag() l = tags.ul() - for server in sorted(per_server.keys(), key=lambda s: s.get_name()): + for server in sorted(list(per_server.keys()), key=lambda s: s.get_name()): times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) - l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) + l(tags.li("[%s]: %s" % (str(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Response Times: ", l) @@ -1208,7 +1220,7 @@ class MapupdateStatusElement(Element): else: times.append("privkey(" + abbreviate_time(t) + ")") times_s = ", ".join(times) - l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) + l(tags.li("[%s]: %s" % (str(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Response Times: ", l) @@ -1298,9 +1310,9 @@ class Status(MultiFormatResource): try: stype, count_s = path.split(b"-") except ValueError: - raise WebError("no '-' in '{}'".format(unicode(path, "utf-8"))) + raise WebError("no '-' in '{}'".format(str(path, "utf-8"))) count = int(count_s) - stype = unicode(stype, "ascii") + stype = str(stype, "ascii") if stype == "up": for s in itertools.chain(h.list_all_upload_statuses(), h.list_all_helper_statuses()): From 0be3349c5d12d9e855f78b52ab971b811c33f494 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 Mar 2021 09:24:49 -0500 Subject: [PATCH 2/7] News file --- newsfragments/3624.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3624.minor diff --git a/newsfragments/3624.minor b/newsfragments/3624.minor new file mode 100644 index 000000000..e69de29bb From 7b1911620f900e05664c734118720f34f6c6ced3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 Mar 2021 09:31:35 -0500 Subject: [PATCH 3/7] Port to Python 3. --- src/allmydata/storage_client.py | 4 ++-- src/allmydata/util/_python3.py | 2 ++ src/allmydata/web/storage.py | 15 +++++++++++---- src/allmydata/web/storage_plugins.py | 10 ++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 75c554562..0986ded06 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -236,11 +236,11 @@ class StorageFarmBroker(service.MultiService): for plugin in getPlugins(IFoolscapStoragePlugin) } - return { + return UnicodeKeyDict({ name: plugins[name].get_client_resource(node_config) for (name, config) in self.storage_client_config.storage_plugins.items() - } + }) @log_call( action_type=u"storage-client:broker:make-storage-server", diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 74aa1cea8..8152c9912 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -128,6 +128,8 @@ PORTED_MODULES = [ "allmydata.web.private", "allmydata.web.root", "allmydata.web.status", + "allmydata.web.storage", + "allmydata.web.storage_plugins", "allmydata.webish", ] diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9452c2a4e..f2f021a15 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,4 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from twisted.python.filepath import FilePath @@ -318,7 +328,4 @@ class StorageStatus(MultiFormatResource): "lease-checker": self._storage.lease_checker.get_state(), "lease-checker-progress": self._storage.lease_checker.get_progress(), } - result = json.dumps(d, indent=1) + "\n" - if PY2: - result = result.decode("utf-8") - return result.encode("utf-8") + return json.dumps(d, indent=1) + "\n" diff --git a/src/allmydata/web/storage_plugins.py b/src/allmydata/web/storage_plugins.py index 939047c6e..41bed9d81 100644 --- a/src/allmydata/web/storage_plugins.py +++ b/src/allmydata/web/storage_plugins.py @@ -1,7 +1,17 @@ """ This module implements a resource which has as children the web resources of all enabled storage client plugins. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.web.resource import ( Resource, From 0647b6368114e2605a8f5b38261269111f651db0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 Mar 2021 09:35:34 -0500 Subject: [PATCH 4/7] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/web/unlinked.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8152c9912..a3b06f7fa 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -130,6 +130,7 @@ PORTED_MODULES = [ "allmydata.web.status", "allmydata.web.storage", "allmydata.web.storage_plugins", + "allmydata.web.unlinked", "allmydata.webish", ] diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index cc13ca1a7..425622496 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -1,4 +1,14 @@ -from past.builtins import unicode +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from urllib.parse import quote as urlquote @@ -119,8 +129,8 @@ class UploadResultsElement(status.UploadResultsRendererMixin): def download_link(self, req, tag): d = self.upload_results() d.addCallback(lambda res: - tags.a("/uri/" + unicode(res.get_uri(), "utf-8"), - href="/uri/" + urlquote(unicode(res.get_uri(), "utf-8")))) + tags.a("/uri/" + str(res.get_uri(), "utf-8"), + href="/uri/" + urlquote(str(res.get_uri(), "utf-8")))) return d From 0125deb6032369ca19e0f098719bc49129eec35f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 09:41:49 -0500 Subject: [PATCH 5/7] Fix typo. --- src/allmydata/mutable/servermap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index d793d1dcd..4f3226649 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -1212,7 +1212,7 @@ class ServermapUpdater(object): break more_queries.append(self.extra_servers.pop(0)) - self.log(format="sending %(more)d more queries: %(who)d", + self.log(format="sending %(more)d more queries: %(who)s", more=len(more_queries), who=" ".join(["[%r]" % s.get_name() for s in more_queries]), level=log.NOISY) From 061f886f4555633259ae519e47163fc715360e1a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 09:42:06 -0500 Subject: [PATCH 6/7] Fix typo. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bc18ed519..33e5830ff 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ deps = # more useful results. usedevelop = False # We use extras=test to get things like "mock" that are required for our unit -# tests.8 +# tests. extras = test setenv = From 437ab584125b554019d9162fdf94f5d339d347d8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 5 Mar 2021 09:45:07 -0500 Subject: [PATCH 7/7] Fix flakes. --- src/allmydata/web/status.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index e66966574..0401fb586 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -547,14 +547,14 @@ class DownloadStatusElement(Element): for r_ev in self._download_status.read_events: start = r_ev["start"] length = r_ev["length"] - bytes = r_ev["bytes_returned"] + bytes_returned = r_ev["bytes_returned"] decrypt_time = "" if bytes: - decrypt_time = self._rate_and_time(bytes, r_ev["decrypt_time"]) + decrypt_time = self._rate_and_time(bytes_returned, r_ev["decrypt_time"]) speed, rtt = "","" if r_ev["finish_time"] is not None: rtt = r_ev["finish_time"] - r_ev["start_time"] - r_ev["paused_time"] - speed = abbreviate_rate(compute_rate(bytes, rtt)) + speed = abbreviate_rate(compute_rate(bytes_returned, rtt)) rtt = abbreviate_time(rtt) paused = abbreviate_time(r_ev["paused_time"]) @@ -562,7 +562,7 @@ class DownloadStatusElement(Element): tags.td("[%d:+%d]" % (start, length)), tags.td(srt(r_ev["start_time"])), tags.td(srt(r_ev["finish_time"])), - tags.td(str(bytes)), + tags.td(str(bytes_returned)), tags.td(rtt), tags.td(decrypt_time), tags.td(paused),