From f656daea7b95636f1fbac461fa184d75850b00cc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:43:56 -0400 Subject: [PATCH 1/7] News file --- newsfragments/3718.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3718.minor diff --git a/newsfragments/3718.minor b/newsfragments/3718.minor new file mode 100644 index 000000000..e69de29bb From 1e0bf545baf8c1380d58a9a362d1a8a3386d685e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:47:07 -0400 Subject: [PATCH 2/7] Port to Python 3. --- src/allmydata/scripts/default_nodedir.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/scripts/default_nodedir.py b/src/allmydata/scripts/default_nodedir.py index c38e20fa7..00924b8f9 100644 --- a/src/allmydata/scripts/default_nodedir.py +++ b/src/allmydata/scripts/default_nodedir.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import six diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 338dd9423..66144d6c9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.scripts.common", "allmydata.scripts.create_node", "allmydata.scripts.debug", + "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", "allmydata.scripts.types_", "allmydata.stats", From bf133be19548359f503439cd78936bcb6aa9ca17 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:56:13 -0400 Subject: [PATCH 3/7] Port to Python 3. --- src/allmydata/scripts/slow_operation.py | 14 +++++++++++--- src/allmydata/util/_python3.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index b4b2f8196..620edac34 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -1,7 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from future.utils import PY3 -from past.builtins import unicode +from future.utils import PY2, PY3 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str import os, time @@ -29,7 +37,7 @@ class SlowOperationRunner(object): except UnknownAliasError as e: e.display(stderr) return 1 - path = unicode(path, "utf-8") + path = str(path, "utf-8") if path == '/': path = '' url = nodeurl + "uri/%s" % url_quote(rootcap) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 66144d6c9..bdbdfa5c0 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -100,6 +100,7 @@ PORTED_MODULES = [ "allmydata.scripts.debug", "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", + "allmydata.scripts.slow_operation", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 1b873126524912c1abc726202b8f9cca66c3be43 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:58:59 -0400 Subject: [PATCH 4/7] Port to Python 3. --- src/allmydata/scripts/tahoe_add_alias.py | 21 ++++++++++++++------- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 19474b9e8..8476aeb28 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -1,7 +1,14 @@ -from __future__ import print_function +""" +Ported to Python 3. +""" from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path import codecs @@ -37,7 +44,7 @@ def add_line_to_aliasfile(aliasfile, alias, cap): def add_alias(options): nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) cap = options.cap stdout = options.stdout stderr = options.stderr @@ -54,7 +61,7 @@ def add_alias(options): show_output(stderr, "Alias {alias} already exists!", alias=alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") - cap = unicode(uri.from_string_dirnode(cap).to_string(), 'utf-8') + cap = str(uri.from_string_dirnode(cap).to_string(), 'utf-8') add_line_to_aliasfile(aliasfile, alias, cap) show_output(stdout, "Alias {alias} added", alias=alias) @@ -64,7 +71,7 @@ def create_alias(options): # mkdir+add_alias nodedir = options['node-directory'] alias = options.alias - precondition(isinstance(alias, unicode), alias=alias) + precondition(isinstance(alias, str), alias=alias) stdout = options.stdout stderr = options.stderr if u":" in alias: @@ -94,7 +101,7 @@ def create_alias(options): # probably check for others.. - add_line_to_aliasfile(aliasfile, alias, unicode(new_uri, "utf-8")) + add_line_to_aliasfile(aliasfile, alias, str(new_uri, "utf-8")) show_output(stdout, "Alias {alias} created", alias=alias) return 0 @@ -114,7 +121,7 @@ def show_output(fp, template, **kwargs): ``encoding`` attribute at all (eg StringIO.StringIO) by writing utf-8-encoded bytes. """ - assert isinstance(template, unicode) + assert isinstance(template, str) # On Python 3 fp has an encoding attribute under all real usage. On # Python 2, the encoding attribute is None if stdio is not a tty. The diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bdbdfa5c0..0bb9286d7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -101,6 +101,7 @@ PORTED_MODULES = [ "allmydata.scripts.default_nodedir", "allmydata.scripts.runner", "allmydata.scripts.slow_operation", + "allmydata.scripts.tahoe_add_alias", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From 693e98e3a795159958def17cc0752e95c923aa9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 10:05:44 -0400 Subject: [PATCH 5/7] Port to Python 3. --- src/allmydata/scripts/tahoe_backup.py | 24 ++++++++++++++++-------- src/allmydata/util/_python3.py | 1 + 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/allmydata/scripts/tahoe_backup.py b/src/allmydata/scripts/tahoe_backup.py index 4847f2e8c..b574f16e8 100644 --- a/src/allmydata/scripts/tahoe_backup.py +++ b/src/allmydata/scripts/tahoe_backup.py @@ -1,6 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os.path import time @@ -293,7 +301,7 @@ def collect_backup_targets(root, listdir, filter_children): yield FilenameUndecodableTarget(root, isdir=True) else: for child in filter_children(children): - assert isinstance(child, unicode), child + assert isinstance(child, str), child childpath = os.path.join(root, child) if os.path.islink(childpath): yield LinkTarget(childpath, isdir=False) @@ -498,10 +506,10 @@ class BackupProgress(object): ) def _format_elapsed(self, elapsed): - seconds = elapsed.total_seconds() - hours = int(seconds / 3600) - minutes = int(seconds / 60 % 60) - seconds = int(seconds % 60) + seconds = int(elapsed.total_seconds()) + hours = seconds // 3600 + minutes = (seconds // 60) % 60 + seconds = seconds % 60 return "{}h {}m {}s".format( hours, minutes, @@ -526,12 +534,12 @@ class BackupProgress(object): return self, { os.path.basename(create_path): create_value for (create_path, create_value) - in self._create_contents.items() + in list(self._create_contents.items()) if os.path.dirname(create_path) == dirpath }, { os.path.basename(compare_path): compare_value for (compare_path, compare_value) - in self._compare_contents.items() + in list(self._compare_contents.items()) if os.path.dirname(compare_path) == dirpath } diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 0bb9286d7..6e9dfd2e4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -102,6 +102,7 @@ PORTED_MODULES = [ "allmydata.scripts.runner", "allmydata.scripts.slow_operation", "allmydata.scripts.tahoe_add_alias", + "allmydata.scripts.tahoe_backup", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From f73f601f67b73535e80457de8304153a8e08dbd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 10:35:16 -0400 Subject: [PATCH 6/7] Port to Python 3. --- src/allmydata/scripts/tahoe_check.py | 32 +++++++++++++++++++--------- src/allmydata/util/_python3.py | 2 ++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py index 82885d073..6bafe3d1a 100644 --- a/src/allmydata/scripts/tahoe_check.py +++ b/src/allmydata/scripts/tahoe_check.py @@ -1,19 +1,26 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_str, ensure_text + from urllib.parse import quote as url_quote import json -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 - from twisted.protocols.basic import LineOnlyReceiver from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \ UnknownAliasError from allmydata.scripts.common_http import do_http, format_http_error -from allmydata.util.encodingutil import quote_output, quote_path +from allmydata.util.encodingutil import quote_output, quote_path, get_io_encoding class Checker(object): pass @@ -168,7 +175,9 @@ class DeepCheckOutput(LineOnlyReceiver, object): # LIT files and directories do not have a "summary" field. summary = cr.get("summary", "Healthy (LIT)") - print("%s: %s" % (quote_path(path), quote_output(summary, quotemarks=False)), file=stdout) + # When Python 2 is dropped the ensure_text()/ensure_str() will be unnecessary. + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), quote_output(summary, quotemarks=False)), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares for shareloc in cr["results"].get("list-corrupt-shares", []): @@ -245,11 +254,14 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object): if not path: path = [""] # we don't seem to have a summary available, so build one + # When Python 2 is dropped the ensure_text/ensure_str crap can be + # dropped. if was_healthy: - summary = "healthy" + summary = ensure_str("healthy") else: - summary = "not healthy" - print("%s: %s" % (quote_path(path), summary), file=stdout) + summary = ensure_str("not healthy") + print(ensure_text(ensure_str("%s: %s") % (quote_path(path), summary), + encoding=get_io_encoding()), file=stdout) # always print out corrupt shares prr = crr.get("pre-repair-results", {}) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 6e9dfd2e4..50e429979 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -16,6 +16,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + PORTED_INTEGRATION_TESTS = [ "integration.test_aaa_aardvark", "integration.test_servers_of_happiness", @@ -103,6 +104,7 @@ PORTED_MODULES = [ "allmydata.scripts.slow_operation", "allmydata.scripts.tahoe_add_alias", "allmydata.scripts.tahoe_backup", + "allmydata.scripts.tahoe_check", "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", From a0744ffa8c834e65153f0e692cd9b47efc4a7450 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Jun 2021 09:54:57 -0400 Subject: [PATCH 7/7] Don't shadow builtin. --- src/allmydata/scripts/slow_operation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py index 620edac34..3c23fb533 100644 --- a/src/allmydata/scripts/slow_operation.py +++ b/src/allmydata/scripts/slow_operation.py @@ -66,10 +66,10 @@ class SlowOperationRunner(object): def wait_for_results(self): last = 0 - for next in self.poll_times(): - delay = next - last + for next_item in self.poll_times(): + delay = next_item - last time.sleep(delay) - last = next + last = next_item if self.poll(): return 0