From c98d90bf74675694cfa56bbcf55710a91b399c20 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:41:49 -0400 Subject: [PATCH 01/21] News file. --- newsfragments/3716.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3716.minor diff --git a/newsfragments/3716.minor b/newsfragments/3716.minor new file mode 100644 index 000000000..e69de29bb From 250ca3170db995c0f8579b239c4692070e0a7d14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:42:13 -0400 Subject: [PATCH 02/21] Port to Python 3. --- src/allmydata/scripts/admin.py | 19 ++++++++++++++----- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index abe3d093c..a9feed0dd 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,6 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_binary try: @@ -25,9 +34,9 @@ def print_keypair(options): from allmydata.crypto import ed25519 out = options.stdout private_key, public_key = ed25519.create_signing_keypair() - print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), file=out) - print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) class DerivePubkeyOptions(BaseOptions): @@ -52,8 +61,8 @@ def derive_pubkey(options): privkey_vs = options.privkey privkey_vs = ensure_binary(privkey_vs) private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs) - print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out) - print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) + print("private:", str(ed25519.string_from_signing_key(private_key), "ascii"), file=out) + print("public:", str(ed25519.string_from_verifying_key(public_key), "ascii"), file=out) return 0 class AdminCommand(BaseOptions): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4bbfc0e68..dbdd8b92b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -88,6 +88,7 @@ PORTED_MODULES = [ "allmydata.node", "allmydata.nodemaker", "allmydata.scripts", + "allmydata.scripts.admin", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From 66f53fada8a8e5d0d5de4140358cb517a2370a6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:46:15 -0400 Subject: [PATCH 03/21] Port to Python 3. --- src/allmydata/scripts/backupdb.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/backupdb.py b/src/allmydata/scripts/backupdb.py index 1bffbbfc3..c7827e56e 100644 --- a/src/allmydata/scripts/backupdb.py +++ b/src/allmydata/scripts/backupdb.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path, sys, time, random, stat from allmydata.util.netstring import netstring diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index dbdd8b92b..510e50d35 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -89,6 +89,7 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", + "allmydata.scripts.backupdb", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From a2280b7660baaa7623faf2c91151819bbebe285a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 09:57:09 -0400 Subject: [PATCH 04/21] Port to Python 3. --- src/allmydata/scripts/cli.py | 18 ++++++++++++++---- src/allmydata/test/cli/test_backup.py | 10 ++++++---- src/allmydata/util/_python3.py | 3 ++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 011dc3b21..41695aa47 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function -from past.builtins import unicode +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os.path, re, fnmatch @@ -38,7 +47,7 @@ class FileStoreOptions(BaseOptions): # compute a node-url from the existing options, put in self['node-url'] if self['node-url']: - if (not isinstance(self['node-url'], (bytes, unicode)) + if (not isinstance(self['node-url'], (bytes, str)) or not NODEURL_RE.match(self['node-url'])): msg = ("--node-url is required to be a string and look like " "\"http://HOSTNAMEORADDR:PORT\", not: %r" % @@ -354,8 +363,9 @@ class BackupOptions(FileStoreOptions): abs_filepath = argv_to_abspath(filepath) try: exclude_file = open(abs_filepath) - except: - raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath)) + except Exception as e: + raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( + quote_local_unicode_path(abs_filepath), e)) try: for line in exclude_file: self.opt_exclude(line) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 4bd4b8bfa..4cc7e865f 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -9,9 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import __builtin__ as builtins -else: - import builtins + import os.path from six.moves import cStringIO as StringIO @@ -430,7 +428,11 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): else: return original_open(name, *args) - patcher = MonkeyPatcher((builtins, 'open', call_file)) + if PY2: + from allmydata.scripts import cli as module_to_patch + else: + import builtins as module_to_patch + patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 510e50d35..7767a52d8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -89,7 +89,8 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", - "allmydata.scripts.backupdb", + "allmydata.scripts.backupdb" + "allmydata.scripts.cli", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From bcf2374dfe0ff83f9014c84519436f31c6b6922d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:27:07 -0400 Subject: [PATCH 05/21] Port to Python 3. --- src/allmydata/scripts/common_http.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index a53f6baa8..bf47f2dd6 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,5 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os from io import BytesIO from six.moves import urllib, http_client diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 7767a52d8..67a93198b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -91,6 +91,7 @@ PORTED_MODULES = [ "allmydata.scripts.admin", "allmydata.scripts.backupdb" "allmydata.scripts.cli", + "allmydata.scripts.common_http", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From e3f079555086e0180df1d9eb8750d7c42658766a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:31:26 -0400 Subject: [PATCH 06/21] Port to Python 3. --- src/allmydata/scripts/common.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index bc969de93..e2f13b262 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,7 +1,19 @@ # coding: utf-8 +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + + import os, sys, textwrap import codecs from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 67a93198b..eee8bf65b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -92,6 +92,7 @@ PORTED_MODULES = [ "allmydata.scripts.backupdb" "allmydata.scripts.cli", "allmydata.scripts.common_http", + "allmydata.scripts.common", "allmydata.scripts.create_node", "allmydata.scripts.runner", "allmydata.scripts.types_", From 7c170317c00a33764b5967e6607384f6fde9afd1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:41:53 -0400 Subject: [PATCH 07/21] Remove some cruft. --- src/allmydata/scripts/common.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index e2f13b262..0a9ab8714 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -12,6 +12,8 @@ from __future__ import print_function from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +else: + from typing import Union import os, sys, textwrap @@ -29,13 +31,6 @@ from yaml import ( safe_dump, ) -# Python 2 compatibility -from future.utils import PY2 -if PY2: - from future.builtins import str # noqa: F401 -else: - from typing import Union - from twisted.python import usage from allmydata.util.assertutil import precondition From 0655b020505529f0182e808d992fd10224f35b52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 14 May 2021 10:50:04 -0400 Subject: [PATCH 08/21] Port to Python 3. --- src/allmydata/scripts/debug.py | 104 ++++++++++++++++++--------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 57 insertions(+), 48 deletions(-) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index ba40519de..2d6ba4602 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,12 +1,20 @@ +""" +Ported to Python 3. +""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division from __future__ import print_function +from future.utils import PY2, bchr +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + try: from allmydata.scripts.types_ import SubCommands except ImportError: pass -from future.utils import bchr -from past.builtins import unicode # do not import any allmydata modules at this level. Do that from inside # individual functions instead. @@ -94,7 +102,7 @@ def dump_immutable_chk_share(f, out, options): def to_string(v): if isinstance(v, bytes): - return unicode(v, "utf-8") + return str(v, "utf-8") else: return str(v) @@ -173,9 +181,9 @@ def format_expiration_time(expiration_time): remains = expiration_time - now when = "%ds" % remains if remains > 24*3600: - when += " (%d days)" % (remains / (24*3600)) + when += " (%d days)" % (remains // (24*3600)) elif remains > 3600: - when += " (%d hours)" % (remains / 3600) + when += " (%d hours)" % (remains // 3600) return when @@ -205,7 +213,7 @@ def dump_mutable_share(options): print(file=out) print("Mutable slot found:", file=out) print(" share_type: %s" % share_type, file=out) - print(" write_enabler: %s" % unicode(base32.b2a(WE), "utf-8"), file=out) + print(" write_enabler: %s" % str(base32.b2a(WE), "utf-8"), file=out) print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out) print(" num_extra_leases: %d" % num_extra_leases, file=out) print(" container_size: %d" % container_size, file=out) @@ -217,8 +225,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.expiration_time) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % unicode(base32.b2a(lease.renew_secret), "utf-8"), file=out) - print(" cancel_secret: %s" % unicode(base32.b2a(lease.cancel_secret), "utf-8"), file=out) + print(" renew_secret: %s" % str(base32.b2a(lease.renew_secret), "utf-8"), file=out) + print(" cancel_secret: %s" % str(base32.b2a(lease.cancel_secret), "utf-8"), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) @@ -266,8 +274,8 @@ def dump_SDMF_share(m, length, options): print(" SDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) - print(" IV: %s" % unicode(base32.b2a(IV), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) + print(" IV: %s" % str(base32.b2a(IV), "utf-8"), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) print(" segsize: %d" % segsize, file=out) @@ -360,7 +368,7 @@ def dump_MDMF_share(m, length, options): print(" MDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) + print(" root_hash: %s" % str(base32.b2a(root_hash), "utf-8"), file=out) #print(" IV: %s" % base32.b2a(IV), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) @@ -485,19 +493,19 @@ def _dump_secrets(storage_index, secret, nodeid, out): if secret: crs = hashutil.my_renewal_secret_hash(secret) - print(" client renewal secret:", unicode(base32.b2a(crs), "ascii"), file=out) + print(" client renewal secret:", str(base32.b2a(crs), "ascii"), file=out) frs = hashutil.file_renewal_secret_hash(crs, storage_index) - print(" file renewal secret:", unicode(base32.b2a(frs), "ascii"), file=out) + print(" file renewal secret:", str(base32.b2a(frs), "ascii"), file=out) if nodeid: renew = hashutil.bucket_renewal_secret_hash(frs, nodeid) - print(" lease renewal secret:", unicode(base32.b2a(renew), "ascii"), file=out) + print(" lease renewal secret:", str(base32.b2a(renew), "ascii"), file=out) ccs = hashutil.my_cancel_secret_hash(secret) - print(" client cancel secret:", unicode(base32.b2a(ccs), "ascii"), file=out) + print(" client cancel secret:", str(base32.b2a(ccs), "ascii"), file=out) fcs = hashutil.file_cancel_secret_hash(ccs, storage_index) - print(" file cancel secret:", unicode(base32.b2a(fcs), "ascii"), file=out) + print(" file cancel secret:", str(base32.b2a(fcs), "ascii"), file=out) if nodeid: cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid) - print(" lease cancel secret:", unicode(base32.b2a(cancel), "ascii"), file=out) + print(" lease cancel secret:", str(base32.b2a(cancel), "ascii"), file=out) def dump_uri_instance(u, nodeid, secret, out, show_header=True): from allmydata import uri @@ -508,19 +516,19 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): if isinstance(u, uri.CHKFileURI): if show_header: print("CHK File:", file=out) - print(" key:", unicode(base32.b2a(u.key), "ascii"), file=out) - print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) + print(" key:", str(base32.b2a(u.key), "ascii"), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.CHKFileVerifierURI): if show_header: print("CHK Verifier URI:", file=out) - print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out) + print(" UEB hash:", str(base32.b2a(u.uri_extension_hash), "ascii"), file=out) print(" size:", u.size, file=out) print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) elif isinstance(u, uri.LiteralFileURI): if show_header: @@ -530,52 +538,52 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True): elif isinstance(u, uri.WriteableSSKFileURI): # SDMF if show_header: print("SDMF Writeable URI:", file=out) - print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlySSKFileURI): if show_header: print("SDMF Read-only URI:", file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.SSKVerifierURI): if show_header: print("SDMF Verifier URI:", file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF if show_header: print("MDMF Writeable URI:", file=out) - print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" writekey:", str(base32.b2a(u.writekey), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) print(file=out) if nodeid: we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid) - print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out) + print(" write_enabler:", str(base32.b2a(we), "ascii"), file=out) print(file=out) _dump_secrets(u.get_storage_index(), secret, nodeid, out) elif isinstance(u, uri.ReadonlyMDMFFileURI): if show_header: print("MDMF Read-only URI:", file=out) - print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" readkey:", str(base32.b2a(u.readkey), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.MDMFVerifierURI): if show_header: print("MDMF Verifier URI:", file=out) - print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out) - print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out) + print(" storage index:", str(si_b2a(u.get_storage_index()), "ascii"), file=out) + print(" fingerprint:", str(base32.b2a(u.fingerprint), "ascii"), file=out) elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory @@ -623,7 +631,7 @@ class FindSharesOptions(BaseOptions): def parseArgs(self, storage_index_s, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath self.si_s = storage_index_s - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) description = """ Locate all shares for the given storage index. This command looks through one @@ -666,7 +674,7 @@ def find_shares(options): class CatalogSharesOptions(BaseOptions): def parseArgs(self, *nodedirs): from allmydata.util.encodingutil import argv_to_abspath - self.nodedirs = map(argv_to_abspath, nodedirs) + self.nodedirs = list(map(argv_to_abspath, nodedirs)) if not nodedirs: raise usage.UsageError("must specify at least one node directory") @@ -753,7 +761,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy @@ -782,7 +790,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, unicode(base32.b2a(root_hash), "utf-8"), + seqnum, str(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) @@ -816,7 +824,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - unicode(ueb_hash, "utf-8"), expiration, + str(ueb_hash, "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: @@ -990,7 +998,7 @@ def fixOptionsClass(args): class FlogtoolOptions(foolscap_cli.Options): def __init__(self): super(FlogtoolOptions, self).__init__() - self.subCommands = map(fixOptionsClass, self.subCommands) + self.subCommands = list(map(fixOptionsClass, self.subCommands)) def getSynopsis(self): return "Usage: tahoe [global-options] debug flogtool COMMAND [flogtool-options]" diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index eee8bf65b..5572a5760 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -94,6 +94,7 @@ PORTED_MODULES = [ "allmydata.scripts.common_http", "allmydata.scripts.common", "allmydata.scripts.create_node", + "allmydata.scripts.debug", "allmydata.scripts.runner", "allmydata.scripts.types_", "allmydata.stats", From c92bf72437a96d383e3cf272365b2f4f8a7cfc2e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 09:36:43 -0400 Subject: [PATCH 09/21] Add missing comma. --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8190949e1..338dd9423 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -92,7 +92,7 @@ PORTED_MODULES = [ "allmydata.nodemaker", "allmydata.scripts", "allmydata.scripts.admin", - "allmydata.scripts.backupdb" + "allmydata.scripts.backupdb", "allmydata.scripts.cli", "allmydata.scripts.common_http", "allmydata.scripts.common", From 073f521cc1ef22b19bf6b1bc2fc94ae6dbd16c73 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 May 2021 12:59:38 -0400 Subject: [PATCH 10/21] WIP --- src/allmydata/test/cli/test_backup.py | 6 ++++-- src/allmydata/util/encodingutil.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 4cc7e865f..06da8a7f1 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -15,13 +15,14 @@ import os.path from six.moves import cStringIO as StringIO from datetime import timedelta import re +import sys from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv +from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.namespace import Namespace from allmydata.scripts import cli, backupdb from ..common_util import StallMixin @@ -372,7 +373,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*".encode(get_io_encoding()) + doc_pattern_arg = quote_output(u"*d\u00F8c*", sys.stdout.encoding) except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.") @@ -399,6 +400,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) + import pdb; pdb.set_trace() self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 637374064..4cf690e65 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -276,7 +276,10 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): On Python 3, returns Unicode strings. """ precondition(isinstance(s, (bytes, unicode)), s) - encoding = encoding or io_encoding + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + encoding = encoding or getattr(sys.stdout, "encoding") or io_encoding if quote_newlines is None: quote_newlines = quotemarks From 0a9b3a4003420b0b117f0e7b1a55a0753b4f0ad4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 May 2021 15:17:57 -0400 Subject: [PATCH 11/21] Don't want pdb. --- src/allmydata/test/cli/test_backup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index c54591644..e9f8087a2 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -409,7 +409,6 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): fileutil.write(excl_filepath, exclusion_string) backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) - import pdb; pdb.set_trace() self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) From 741af0b0eb8b8fe3869f32154d607e1545e1a62f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 26 May 2021 15:27:16 -0400 Subject: [PATCH 12/21] Fix imports. --- src/allmydata/test/cli/test_backup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index e9f8087a2..6ac55a9e8 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -9,9 +9,6 @@ from __future__ import unicode_literals from future.utils import PY2, PY3 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import __builtin__ as builtins -else: - import builtins from six import ensure_str import os.path @@ -25,7 +22,7 @@ from twisted.python.monkey import MonkeyPatcher from allmydata.util import fileutil from allmydata.util.fileutil import abspath_expanduser_unicode -from allmydata.util.encodingutil import quote_output, unicode_to_argv +from allmydata.util.encodingutil import get_io_encoding, unicode_to_argv from allmydata.util.namespace import Namespace from allmydata.scripts import cli, backupdb from ..common_util import StallMixin From d78e9d891d37b752296d60a3bc850f483850e551 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 11:40:59 -0400 Subject: [PATCH 13/21] Passing test. --- src/allmydata/scripts/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 41695aa47..922079c92 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -362,11 +362,12 @@ class BackupOptions(FileStoreOptions): line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = open(abs_filepath) + exclude_file = open(abs_filepath, "rb") except Exception as e: raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( quote_local_unicode_path(abs_filepath), e)) try: + for line in exclude_file: self.opt_exclude(line) finally: From 49be6906c8628b862d6ca751c0174c69be248fd9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:07:56 -0400 Subject: [PATCH 14/21] Fix test. --- src/allmydata/test/cli/test_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py index e01dcc4cb..472105ca1 100644 --- a/src/allmydata/test/cli/test_check.py +++ b/src/allmydata/test/cli/test_check.py @@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO from allmydata import uri from allmydata.util import base32 -from allmydata.util.encodingutil import to_bytes +from allmydata.util.encodingutil import to_bytes, quote_output_u from allmydata.mutable.publish import MutableData from allmydata.immutable import upload from allmydata.scripts import debug @@ -168,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase): self.uris = {} self.fileurls = {} DATA = b"data" * 100 - quoted_good = u"'g\u00F6\u00F6d'" + quoted_good = quote_output_u("g\u00F6\u00F6d") d = c0.create_dirnode() def _stash_root_and_create_file(n): From 0425b64041651ba385eadbb07dcd2d8081e82473 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:23 -0400 Subject: [PATCH 15/21] Match current logic. --- src/allmydata/test/test_encodingutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_encodingutil.py b/src/allmydata/test/test_encodingutil.py index f7987d466..062c64ba1 100644 --- a/src/allmydata/test/test_encodingutil.py +++ b/src/allmydata/test/test_encodingutil.py @@ -379,7 +379,10 @@ class QuoteOutput(ReallyEqualMixin, unittest.TestCase): check(u"\n", u"\"\\x0a\"", quote_newlines=True) def test_quote_output_default(self): - self.test_quote_output_utf8(None) + """Default is the encoding of sys.stdout if known, otherwise utf-8.""" + encoding = getattr(sys.stdout, "encoding") or "utf-8" + self.assertEqual(quote_output(u"\u2621"), + quote_output(u"\u2621", encoding=encoding)) def win32_other(win32, other): From 8aa339127638b413a943727d2985a5bd7245487c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:46 -0400 Subject: [PATCH 16/21] Consistent behavior. --- src/allmydata/test/common_util.py | 3 ++- src/allmydata/util/encodingutil.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 9e7f0618c..21b8817a4 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -12,6 +12,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os +import sys import time import signal from random import randrange @@ -85,7 +86,7 @@ def run_cli_native(verb, *args, **kwargs): bytes. """ nodeargs = kwargs.pop("nodeargs", []) - encoding = kwargs.pop("encoding", None) or "utf-8" + encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" return_bytes = kwargs.pop("return_bytes", False) verb = maybe_unicode_to_argv(verb) args = [maybe_unicode_to_argv(a) for a in args] diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 4cf690e65..576091539 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -256,7 +256,11 @@ def quote_output_u(*args, **kwargs): result = quote_output(*args, **kwargs) if isinstance(result, unicode): return result - return result.decode(kwargs.get("encoding", None) or io_encoding) + # Since we're quoting, the assumption is this will be read by a human, and + # therefore printed, so stdout's encoding is the plausible one. io_encoding + # is now always utf-8. + return result.decode(kwargs.get("encoding", None) or + getattr(sys.stdout, "encoding") or io_encoding) def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): @@ -287,7 +291,7 @@ def quote_output(s, quotemarks=True, quote_newlines=None, encoding=None): def _encode(s): if isinstance(s, bytes): try: - s = s.decode('utf-8') + s = s.decode("utf-8") except UnicodeDecodeError: return b'b"%s"' % (ESCAPABLE_8BIT.sub(lambda m: _bytes_escape(m, quote_newlines), s),) From eddd9550f085d0d75fdbef3fdb26c17284ba8154 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:26:56 -0400 Subject: [PATCH 17/21] Pass test. --- src/allmydata/test/cli/test_create_alias.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 6f6881860..99590a631 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -20,7 +20,7 @@ from allmydata.util import fileutil from allmydata.scripts.common import get_aliases from allmydata.scripts import cli, runner from ..no_network import GridTestMixin -from allmydata.util.encodingutil import quote_output +from allmydata.util.encodingutil import quote_output_u from .common import CLITestMixin class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -182,7 +182,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase): (rc, out, err) = args self.failUnlessReallyEqual(rc, 0) self.assertEqual(len(err), 0, err) - self.failUnlessIn(u"Alias %s created" % ensure_text(quote_output(etudes_arg)), out) + self.failUnlessIn(u"Alias %s created" % (quote_output_u(etudes_arg),), out) aliases = get_aliases(self.get_clientdir()) self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:")) From 8be4acd7ec27861d015cfd33e64723180a38937b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:54:27 -0400 Subject: [PATCH 18/21] Faking sys.stdout doesn't go well since we lookup encoding attribute. --- src/allmydata/test/cli/test_status.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 4488299b2..58818ed41 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -130,9 +130,10 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): d.addCallback(_check) return d - @mock.patch('sys.stdout') - def test_help(self, fake): - return self.do_cli('status', '--help') + @defer.inlineCallbacks + def test_help(self): + rc, _, _ = yield self.do_cli('status', '--help') + self.assertEqual(rc, 0) class CommandStatus(unittest.TestCase): From 69c8305ae8837a4746e681385ffd12671c10a023 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 May 2021 13:55:29 -0400 Subject: [PATCH 19/21] Flake fix. --- src/allmydata/test/cli/test_create_alias.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py index 99590a631..176bf7576 100644 --- a/src/allmydata/test/cli/test_create_alias.py +++ b/src/allmydata/test/cli/test_create_alias.py @@ -10,7 +10,6 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_text from six.moves import StringIO import os.path from twisted.trial import unittest From 64f6ccd17ffb53d9a71d562fdbe93b69fed39e83 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 May 2021 09:53:28 -0400 Subject: [PATCH 20/21] Make --exclude-from behavior consistent, at the cost of a minor incompatibility. --- docs/frontends/CLI.rst | 6 +++--- newsfragments/3716.incompat | 1 + newsfragments/3716.minor | 0 src/allmydata/scripts/cli.py | 5 ++--- src/allmydata/test/cli/test_backup.py | 20 ++++++++------------ 5 files changed, 14 insertions(+), 18 deletions(-) create mode 100644 newsfragments/3716.incompat delete mode 100644 newsfragments/3716.minor diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 0badede98..d501b7af5 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -514,10 +514,10 @@ Command Examples the pattern will be matched against any level of the directory tree; it's still impossible to specify absolute path exclusions. -``tahoe backup --exclude-from=/path/to/filename ~ work:backups`` +``tahoe backup --exclude-from-utf-8=/path/to/filename ~ work:backups`` - ``--exclude-from`` is similar to ``--exclude``, but reads exclusion - patterns from ``/path/to/filename``, one per line. + ``--exclude-from-utf-8`` is similar to ``--exclude``, but reads exclusion + patterns from a UTF-8-encoded ``/path/to/filename``, one per line. ``tahoe backup --exclude-vcs ~ work:backups`` diff --git a/newsfragments/3716.incompat b/newsfragments/3716.incompat new file mode 100644 index 000000000..aa03eea47 --- /dev/null +++ b/newsfragments/3716.incompat @@ -0,0 +1 @@ +tahoe backup's --exclude-from has been renamed to --exclude-from-utf-8, and correspondingly requires the file to be UTF-8 encoded. \ No newline at end of file diff --git a/newsfragments/3716.minor b/newsfragments/3716.minor deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 922079c92..c58b5da41 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -357,17 +357,16 @@ class BackupOptions(FileStoreOptions): exclude = self['exclude'] exclude.add(g) - def opt_exclude_from(self, filepath): + def opt_exclude_from_utf_8(self, filepath): """Ignore file matching glob patterns listed in file, one per line. The file is assumed to be in the argv encoding.""" abs_filepath = argv_to_abspath(filepath) try: - exclude_file = open(abs_filepath, "rb") + exclude_file = open(abs_filepath, "r", encoding="utf-8") except Exception as e: raise BackupConfigurationError('Error opening exclude file %s. (Error: %s)' % ( quote_local_unicode_path(abs_filepath), e)) try: - for line in exclude_file: self.opt_exclude(line) finally: diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index 6ac55a9e8..dc5c3ff17 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -355,14 +355,14 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): exclusion_string = "_darcs\n*py\n.svn" excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(subdir_listdir)) self._check_filtering(filtered, subdir_listdir, (u'another_doc.lyx', u'CVS'), (u'.svn', u'_darcs', u'run_snake_run.py')) # test BackupConfigurationError self.failUnlessRaises(cli.BackupConfigurationError, parse, - ['--exclude-from', excl_filepath + '.no', 'from', 'to']) + ['--exclude-from-utf-8', excl_filepath + '.no', 'from', 'to']) # test that an iterator works too backup_options = parse(['--exclude', '*lyx', 'from', 'to']) @@ -373,7 +373,7 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): def test_exclude_options_unicode(self): nice_doc = u"nice_d\u00F8c.lyx" try: - doc_pattern_arg = u"*d\u00F8c*" + doc_pattern_arg_unicode = doc_pattern_arg = u"*d\u00F8c*" if PY2: doc_pattern_arg = doc_pattern_arg.encode(get_io_encoding()) except UnicodeEncodeError: @@ -397,14 +397,10 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) # read exclude patterns from file - exclusion_string = doc_pattern_arg + ensure_str("\nlib.?") - if PY3: - # On Python 2 this gives some garbage encoding. Also on Python 2 we - # expect exclusion string to be bytes. - exclusion_string = exclusion_string.encode(locale.getpreferredencoding(False)) + exclusion_string = (doc_pattern_arg_unicode + "\nlib.?").encode("utf-8") excl_filepath = os.path.join(basedir, 'exclusion') fileutil.write(excl_filepath, exclusion_string) - backup_options = parse(['--exclude-from', excl_filepath, 'from', 'to']) + backup_options = parse(['--exclude-from-utf-8', excl_filepath, 'from', 'to']) filtered = list(backup_options.filter_listdir(root_listdir)) self._check_filtering(filtered, root_listdir, (u'_darcs', u'subdir'), (nice_doc, u'lib.a')) @@ -427,20 +423,20 @@ class Backup(GridTestMixin, CLITestMixin, StallMixin, unittest.TestCase): ns = Namespace() ns.called = False original_open = open - def call_file(name, *args): + def call_file(name, *args, **kwargs): if name.endswith("excludes.dummy"): ns.called = True self.failUnlessEqual(name, abspath_expanduser_unicode(exclude_file)) return StringIO() else: - return original_open(name, *args) + return original_open(name, *args, **kwargs) if PY2: from allmydata.scripts import cli as module_to_patch else: import builtins as module_to_patch patcher = MonkeyPatcher((module_to_patch, 'open', call_file)) - patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to']) + patcher.runWithPatches(parse_options, basedir, "backup", ['--exclude-from-utf-8', unicode_to_argv(exclude_file), 'from', 'to']) self.failUnless(ns.called) def test_ignore_symlinks(self): From b509ff69fcb77088990ff8126da6e9a8c8485374 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 May 2021 11:25:16 -0400 Subject: [PATCH 21/21] Flake fix. --- src/allmydata/test/cli/test_backup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/cli/test_backup.py b/src/allmydata/test/cli/test_backup.py index dc5c3ff17..df598b811 100644 --- a/src/allmydata/test/cli/test_backup.py +++ b/src/allmydata/test/cli/test_backup.py @@ -6,16 +6,14 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, PY3 +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from six import ensure_str import os.path from six.moves import cStringIO as StringIO from datetime import timedelta import re -import locale from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher