From 6035a4a2aee7f6533afba811746cca94e9af4bb3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 13 Apr 2021 09:22:33 -0400
Subject: [PATCH 01/81] News file.

---
 newsfragments/3675.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3675.minor

diff --git a/newsfragments/3675.minor b/newsfragments/3675.minor
new file mode 100644
index 000000000..e69de29bb

From cbd816fbd5909883c4af12b3803c66e47af1344a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 13 Apr 2021 09:34:55 -0400
Subject: [PATCH 02/81] Ensure warnings get turned into exceptions. Getting
 sneaking suspicion it's passing in filenames sometimes, not just modules.

---
 src/allmydata/test/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py
index c75f8d003..e9c47bd69 100644
--- a/src/allmydata/test/__init__.py
+++ b/src/allmydata/test/__init__.py
@@ -36,7 +36,7 @@ from foolscap.logging.incident import IncidentQualifier
 if PY3:
     # Error on BytesWarnings, to catch things like str(b""), but only for
     # allmydata code.
-    warnings.filterwarnings("error", category=BytesWarning, module="allmydata.*")
+    warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")
 
 
 class NonQualifier(IncidentQualifier, object):

From 2299c2dcc827e02c6649913ac2d210bb920d60dc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 13 Apr 2021 09:40:24 -0400
Subject: [PATCH 03/81] Fix implicit str(bytesobj).

---
 src/allmydata/test/test_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index f6a7c2ee1..c80b5dc9c 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -515,7 +515,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
             0,
             "Expected error message from '{}', got something else: {}".format(
                 description,
-                p.get_buffered_output(),
+                str(p.get_buffered_output(), "utf-8"),
             ),
         )
 

From 953c06a18d7d360b84ae1c45216a140efbd6a8ef Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 13 Apr 2021 09:53:08 -0400
Subject: [PATCH 04/81] Fix some plain-str()-of-bytes bugs.

---
 src/allmydata/test/test_download.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py
index 3a42b0819..d61942839 100644
--- a/src/allmydata/test/test_download.py
+++ b/src/allmydata/test/test_download.py
@@ -1304,7 +1304,7 @@ class MyShare(object):
         self._dyhb_rtt = rtt
 
     def __repr__(self):
-        return "sh%d-on-%s" % (self._shnum, self._server.get_name())
+        return "sh%d-on-%s" % (self._shnum, str(self._server.get_name(), "ascii"))
 
 class MySegmentFetcher(SegmentFetcher):
     def __init__(self, *args, **kwargs):
@@ -1383,7 +1383,7 @@ class Selection(unittest.TestCase):
             self.failUnless(node.failed)
             self.failUnless(node.failed.check(NotEnoughSharesError))
             sname = serverA.get_name()
-            self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused="  % sname,
+            self.failUnlessIn("complete= pending=sh0-on-%s overdue= unused="  % str(sname, "ascii"),
                               str(node.failed))
         d.addCallback(_check2)
         return d
@@ -1605,7 +1605,7 @@ class Selection(unittest.TestCase):
             self.failUnless(node.failed)
             self.failUnless(node.failed.check(NotEnoughSharesError))
             sname = servers[b"peer-2"].get_name()
-            self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % sname,
+            self.failUnlessIn("complete=sh0 pending= overdue=sh2-on-%s unused=" % str(sname, "ascii"),
                               str(node.failed))
         d.addCallback(_check4)
         return d

From e242bf50c77d155970fcb488d690f9f4ae44302c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 14 Apr 2021 09:55:21 -0400
Subject: [PATCH 05/81] Handle bytes in log messages.

---
 newsfragments/3626.minor            |  0
 src/allmydata/test/web/test_logs.py | 10 +++++++---
 src/allmydata/web/logs.py           |  4 ++--
 3 files changed, 9 insertions(+), 5 deletions(-)
 create mode 100644 newsfragments/3626.minor

diff --git a/newsfragments/3626.minor b/newsfragments/3626.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py
index 5d697f910..d4fa5e944 100644
--- a/src/allmydata/test/web/test_logs.py
+++ b/src/allmydata/test/web/test_logs.py
@@ -92,7 +92,7 @@ class TestStreamingLogs(unittest.TestCase):
     @inlineCallbacks
     def test_one_log(self):
         """
-        write a single Eliot log and see it streamed via websocket
+        Write a single Eliot log actin and see it streamed via websocket.
         """
 
         proto = yield self.agent.open(
@@ -106,14 +106,18 @@ class TestStreamingLogs(unittest.TestCase):
         proto.on("message", got_message)
 
         @log_call(action_type=u"test:cli:some-exciting-action")
-        def do_a_thing():
+        def do_a_thing(arguments):
             pass
 
-        do_a_thing()
+        do_a_thing(arguments=[u"hello", b"good-day", 123, {"a": 35}, [None]])
 
         proto.transport.loseConnection()
         yield proto.is_closed
 
         self.assertEqual(len(messages), 2)
+        self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action")
+        self.assertEqual(messages[0]["arguments"],
+                         ["hello", "good-day", 123, {"a": 35}, [None]])
+        self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action")
         self.assertEqual("started", messages[0]["action_status"])
         self.assertEqual("succeeded", messages[1]["action_status"])
diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py
index a78e9cd12..9bd59ae53 100644
--- a/src/allmydata/web/logs.py
+++ b/src/allmydata/web/logs.py
@@ -8,8 +8,6 @@ from __future__ import (
     division,
 )
 
-import json
-
 from autobahn.twisted.resource import WebSocketResource
 from autobahn.twisted.websocket import (
     WebSocketServerFactory,
@@ -21,6 +19,8 @@ from twisted.web.resource import (
     Resource,
 )
 
+from allmydata.util import jsonbytes as json
+
 
 class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol):
     """

From 1abf944dd21b8c5db052215b4497696ce2eeb592 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 14 Apr 2021 10:38:57 -0400
Subject: [PATCH 06/81] News file.

---
 newsfragments/3672.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3672.minor

diff --git a/newsfragments/3672.minor b/newsfragments/3672.minor
new file mode 100644
index 000000000..e69de29bb

From 32607b5ada8065f181c0e83937eaf247d9409256 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 14 Apr 2021 10:42:01 -0400
Subject: [PATCH 07/81] For logging, using a new JSON bytes encoder that works
 on any bytes string, not just UTF-8-encoded strings.

---
 src/allmydata/test/__init__.py       |  4 +-
 src/allmydata/test/eliotutil.py      |  7 +--
 src/allmydata/test/test_eliotutil.py |  4 +-
 src/allmydata/test/test_util.py      | 37 +++++++++++--
 src/allmydata/util/eliotutil.py      |  6 +--
 src/allmydata/util/jsonbytes.py      | 78 ++++++++++++++++++++--------
 src/allmydata/web/logs.py            |  5 +-
 7 files changed, 99 insertions(+), 42 deletions(-)

diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py
index c75f8d003..26b17e997 100644
--- a/src/allmydata/test/__init__.py
+++ b/src/allmydata/test/__init__.py
@@ -131,5 +131,5 @@ if sys.platform == "win32":
     initialize()
 
 from eliot import to_file
-from allmydata.util.jsonbytes import BytesJSONEncoder
-to_file(open("eliot.log", "wb"), encoder=BytesJSONEncoder)
+from allmydata.util.jsonbytes import AnyBytesJSONEncoder
+to_file(open("eliot.log", "wb"), encoder=AnyBytesJSONEncoder)
diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index c2359f132..35dfb09eb 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -54,7 +54,7 @@ from twisted.python.monkey import (
     MonkeyPatcher,
 )
 
-from ..util.jsonbytes import BytesJSONEncoder
+from ..util.jsonbytes import AnyBytesJSONEncoder
 
 
 _NAME = Field.for_types(
@@ -73,10 +73,7 @@ RUN_TEST = ActionType(
 
 # On Python 3, we want to use our custom JSON encoder when validating messages
 # can be encoded to JSON:
-if PY2:
-    _memory_logger = MemoryLogger
-else:
-    _memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder)
+_memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder)
 
 
 @attr.s
diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py
index aca677323..3f915ecd2 100644
--- a/src/allmydata/test/test_eliotutil.py
+++ b/src/allmydata/test/test_eliotutil.py
@@ -69,7 +69,7 @@ from ..util.eliotutil import (
     _parse_destination_description,
     _EliotLogging,
 )
-from ..util.jsonbytes import BytesJSONEncoder
+from ..util.jsonbytes import AnyBytesJSONEncoder
 
 from .common import (
     SyncTestCase,
@@ -109,7 +109,7 @@ class ParseDestinationDescriptionTests(SyncTestCase):
         reactor = object()
         self.assertThat(
             _parse_destination_description("file:-")(reactor),
-            Equals(FileDestination(stdout, encoder=BytesJSONEncoder)),
+            Equals(FileDestination(stdout, encoder=AnyBytesJSONEncoder)),
         )
 
 
diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py
index 9887897cf..8f3a39670 100644
--- a/src/allmydata/test/test_util.py
+++ b/src/allmydata/test/test_util.py
@@ -495,10 +495,10 @@ class YAML(unittest.TestCase):
 
 
 class JSONBytes(unittest.TestCase):
-    """Tests for BytesJSONEncoder."""
+    """Tests for jsonbytes module."""
 
     def test_encode_bytes(self):
-        """BytesJSONEncoder can encode bytes.
+        """jsonbytes.dumps() encodes bytes.
 
         Bytes are presumed to be UTF-8 encoded.
         """
@@ -515,7 +515,7 @@ class JSONBytes(unittest.TestCase):
         self.assertEqual(jsonbytes.loads(encoded), expected)
 
     def test_encode_unicode(self):
-        """BytesJSONEncoder encodes Unicode string as usual."""
+        """jsonbytes.dumps() encodes Unicode string as usual."""
         expected = {
             u"hello": [1, u"cd"],
         }
@@ -529,6 +529,37 @@ class JSONBytes(unittest.TestCase):
         self.assertIsInstance(encoded, bytes)
         self.assertEqual(json.loads(encoded, encoding="utf-8"), x)
 
+    def test_any_bytes_unsupported_by_default(self):
+        """By default non-UTF-8 bytes raise error."""
+        bytestring = b"abc\xff\x00"
+        with self.assertRaises(UnicodeDecodeError):
+            jsonbytes.dumps(bytestring)
+        with self.assertRaises(UnicodeDecodeError):
+            jsonbytes.dumps_bytes(bytestring)
+        with self.assertRaises(UnicodeDecodeError):
+            json.dumps(bytestring, cls=jsonbytes.UTF8BytesJSONEncoder)
+
+    def test_any_bytes(self):
+        """If any_bytes is True, non-UTF-8 bytes don't break encoding."""
+        bytestring = b"abc\xff"
+        o = {bytestring: bytestring}
+        expected = {"abc\\xff": "abc\\xff"}
+        self.assertEqual(
+            json.loads(jsonbytes.dumps(o, any_bytes=True)),
+            expected,
+        )
+        self.assertEqual(
+            json.loads(json.dumps(
+                o, cls=jsonbytes.AnyBytesJSONEncoder)),
+            expected,
+        )
+        self.assertEqual(
+            json.loads(jsonbytes.dumps(o, any_bytes=True),
+                       encoding="utf-8"),
+            expected,
+        )
+
+
 
 class FakeGetVersion(object):
     """Emulate an object with a get_version."""
diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py
index 5d144eb1d..4e48fbb9f 100644
--- a/src/allmydata/util/eliotutil.py
+++ b/src/allmydata/util/eliotutil.py
@@ -87,7 +87,7 @@ from twisted.internet.defer import (
 )
 from twisted.application.service import Service
 
-from .jsonbytes import BytesJSONEncoder
+from .jsonbytes import AnyBytesJSONEncoder
 
 
 def validateInstanceOf(t):
@@ -306,7 +306,7 @@ class _DestinationParser(object):
                     rotateLength=rotate_length,
                     maxRotatedFiles=max_rotated_files,
                 )
-        return lambda reactor: FileDestination(get_file(), BytesJSONEncoder)
+        return lambda reactor: FileDestination(get_file(), AnyBytesJSONEncoder)
 
 
 _parse_destination_description = _DestinationParser().parse
@@ -333,4 +333,4 @@ def log_call_deferred(action_type):
 if PY2:
     capture_logging = eliot_capture_logging
 else:
-    capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder)
+    capture_logging = partial(eliot_capture_logging, encoder_=AnyBytesJSONEncoder)
diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index c46a932d0..849fd6f0a 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -16,43 +16,75 @@ if PY2:
 import json
 
 
-def _bytes_to_unicode(obj):
-    """Convert any bytes objects to unicode, recursively."""
-    if isinstance(obj, bytes):
-        return obj.decode("utf-8")
-    if isinstance(obj, dict):
-        new_obj = {}
-        for k, v in obj.items():
-            if isinstance(k, bytes):
-                k = k.decode("utf-8")
-            v = _bytes_to_unicode(v)
-            new_obj[k] = v
-        return new_obj
-    if isinstance(obj, (list, set, tuple)):
-        return [_bytes_to_unicode(i) for i in obj]
-    return obj
+def _make_bytes_to_unicode(any_bytes):
+    """Create a function that recursively converts bytes to unicode.
 
-
-class BytesJSONEncoder(json.JSONEncoder):
+    :param any_bytes: If True, also support non-UTF-8-encoded bytes.
     """
-    A JSON encoder than can also encode bytes.
+    errors = "backslashreplace" if any_bytes else "strict"
 
-    The bytes are assumed to be UTF-8 encoded Unicode strings.
+    def _bytes_to_unicode(obj):
+        """Convert any bytes objects to unicode, recursively."""
+        if isinstance(obj, bytes):
+            return obj.decode("utf-8", errors=errors)
+        if isinstance(obj, dict):
+            new_obj = {}
+            for k, v in obj.items():
+                if isinstance(k, bytes):
+                    k = k.decode("utf-8", errors=errors)
+                v = _bytes_to_unicode(v)
+                new_obj[k] = v
+            return new_obj
+        if isinstance(obj, (list, set, tuple)):
+            return [_bytes_to_unicode(i) for i in obj]
+        return obj
+
+    return _bytes_to_unicode
+
+
+class UTF8BytesJSONEncoder(json.JSONEncoder):
+    """
+    A JSON encoder than can also encode UTF-8 encoded strings.
     """
     def iterencode(self, o, **kwargs):
-        return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs)
+        return json.JSONEncoder.iterencode(
+            self, _make_bytes_to_unicode(False)(o), **kwargs)
+
+
+class AnyBytesJSONEncoder(json.JSONEncoder):
+    """
+    A JSON encoder than can also encode bytes of any sort.
+
+    Bytes are decoded to strings using UTF-8, if that fails to decode then the
+    bytes are quoted.
+    """
+    def iterencode(self, o, **kwargs):
+        return json.JSONEncoder.iterencode(
+            self, _make_bytes_to_unicode(True)(o), **kwargs)
 
 
 def dumps(obj, *args, **kwargs):
     """Encode to JSON, supporting bytes as keys or values.
 
-    The bytes are assumed to be UTF-8 encoded Unicode strings.
+    :param bool any_bytes: If False (the default) the bytes are assumed to be
+        UTF-8 encoded Unicode strings.  If True, non-UTF-8 bytes are quoted for
+        human consumption.
     """
-    return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
+    any_bytes = kwargs.pop("any_bytes", False)
+    if any_bytes:
+        cls = AnyBytesJSONEncoder
+    else:
+        cls = UTF8BytesJSONEncoder
+    return json.dumps(obj, cls=cls, *args, **kwargs)
 
 
 def dumps_bytes(obj, *args, **kwargs):
-    """Encode to JSON, then encode as bytes."""
+    """Encode to JSON, then encode as bytes.
+
+    :param bool all_bytes: If False (the default) the bytes are assumed to be
+        UTF-8 encoded Unicode strings.  If True, non-UTF-8 bytes are quoted for
+        human consumption.
+    """
     result = dumps(obj, *args, **kwargs)
     if PY3:
         result = result.encode("utf-8")
diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py
index 9bd59ae53..a79440eb9 100644
--- a/src/allmydata/web/logs.py
+++ b/src/allmydata/web/logs.py
@@ -47,10 +47,7 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol):
         """
         # probably want a try/except around here? what do we do if
         # transmission fails or anything else bad happens?
-        encoded = json.dumps(message)
-        if isinstance(encoded, str):
-            # On Python 3 dumps() returns Unicode...
-            encoded = encoded.encode("utf-8")
+        encoded = json.dumps_bytes(message, any_bytes=True)
         self.sendMessage(encoded)
 
     def onOpen(self):

From d60bc2841abe03ff459f9c9c4687da5da2c2ff9d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 14 Apr 2021 11:19:04 -0400
Subject: [PATCH 08/81] Oh right, Python 2 Eliot doesn't support custom JSON
 encoders.

---
 src/allmydata/test/eliotutil.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index 35dfb09eb..1685744fd 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -73,7 +73,10 @@ RUN_TEST = ActionType(
 
 # On Python 3, we want to use our custom JSON encoder when validating messages
 # can be encoded to JSON:
-_memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder)
+if PY2:
+    _memory_logger = MemoryLogger
+else:
+    _memory_logger = lambda: MemoryLogger(encoder=AnyBytesJSONEncoder)
 
 
 @attr.s

From 51ebbae15a0a97c0784d7b4863d25d660bc04e89 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 11:21:47 -0400
Subject: [PATCH 09/81] Fix typo.

---
 src/allmydata/test/web/test_logs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py
index d4fa5e944..579e51dbc 100644
--- a/src/allmydata/test/web/test_logs.py
+++ b/src/allmydata/test/web/test_logs.py
@@ -92,7 +92,7 @@ class TestStreamingLogs(unittest.TestCase):
     @inlineCallbacks
     def test_one_log(self):
         """
-        Write a single Eliot log actin and see it streamed via websocket.
+        Write a single Eliot log action and see it streamed via websocket.
         """
 
         proto = yield self.agent.open(

From bc9e4ac72859c7e79ccce3c18158f6721ee5a269 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 11:36:53 -0400
Subject: [PATCH 10/81] Support quoting any-old-bytes correctly on Python 2.

---
 src/allmydata/test/test_util.py |  4 ++--
 src/allmydata/util/jsonbytes.py | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py
index 8f3a39670..4c2e98683 100644
--- a/src/allmydata/test/test_util.py
+++ b/src/allmydata/test/test_util.py
@@ -541,9 +541,9 @@ class JSONBytes(unittest.TestCase):
 
     def test_any_bytes(self):
         """If any_bytes is True, non-UTF-8 bytes don't break encoding."""
-        bytestring = b"abc\xff"
+        bytestring = b"abc\xff\xff123"
         o = {bytestring: bytestring}
-        expected = {"abc\\xff": "abc\\xff"}
+        expected = {"abc\\xff\\xff123": "abc\\xff\\xff123"}
         self.assertEqual(
             json.loads(jsonbytes.dumps(o, any_bytes=True)),
             expected,
diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index 849fd6f0a..995165ee6 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -14,7 +14,18 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import json
+import codecs
 
+if PY2:
+    def backslashreplace_py2(ex):
+        """
+        On Python 2 'backslashreplace' error handler doesn't work, so write our
+        own.
+        """
+        return ''.join('\\x{:02x}'.format(ord(c))
+                       for c in ex.object[ex.start:ex.end]), ex.end
+
+    codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2)
 
 def _make_bytes_to_unicode(any_bytes):
     """Create a function that recursively converts bytes to unicode.
@@ -22,6 +33,8 @@ def _make_bytes_to_unicode(any_bytes):
     :param any_bytes: If True, also support non-UTF-8-encoded bytes.
     """
     errors = "backslashreplace" if any_bytes else "strict"
+    if PY2 and errors == "backslashreplace":
+        errors = "backslashreplace_tahoe_py2"
 
     def _bytes_to_unicode(obj):
         """Convert any bytes objects to unicode, recursively."""

From 61506f87bb1f46f46eac30577be5df9413526883 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 11:55:20 -0400
Subject: [PATCH 11/81] Make BytesWarning->exception global, to ease use in
 integration tests.

---
 src/allmydata/__init__.py      | 17 ++++++++++++++++-
 src/allmydata/test/__init__.py |  6 ------
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py
index b29868c05..333394fc5 100644
--- a/src/allmydata/__init__.py
+++ b/src/allmydata/__init__.py
@@ -8,7 +8,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2
+from future.utils import PY2, PY3
 if PY2:
     # Don't import future str() so we don't break Foolscap serialization on Python 2.
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
@@ -62,3 +62,18 @@ standard_library.install_aliases()
 from ._monkeypatch import patch
 patch()
 del patch
+
+
+# On Python 3, turn BytesWarnings into exceptions. This can have potential
+# production impact... if BytesWarnings are actually present in the codebase.
+# Given that this has been enabled before Python 3 Tahoe-LAFS was publicly
+# released, no such code should exist, and this will ensure it doesn't get
+# added either.
+#
+# Also note that BytesWarnings only happen if Python is run with -b option, so
+# in practice this should only affect tests.
+if PY3:
+    import warnings
+    # Error on BytesWarnings, to catch things like str(b""), but only for
+    # allmydata code.
+    warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")
diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py
index e9c47bd69..45536a6c6 100644
--- a/src/allmydata/test/__init__.py
+++ b/src/allmydata/test/__init__.py
@@ -24,7 +24,6 @@ from future.utils import PY2, PY3
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-import warnings
 from traceback import extract_stack, format_list
 
 from foolscap.pb import Listener
@@ -33,11 +32,6 @@ from twisted.application import service
 
 from foolscap.logging.incident import IncidentQualifier
 
-if PY3:
-    # Error on BytesWarnings, to catch things like str(b""), but only for
-    # allmydata code.
-    warnings.filterwarnings("error", category=BytesWarning, module=".*allmydata.*")
-
 
 class NonQualifier(IncidentQualifier, object):
     def check_event(self, ev):

From fa46efdb3aae72df3f0ecc2987d42b70c92927c7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 11:58:37 -0400
Subject: [PATCH 12/81] Enable BytesWarnings in integration tests.

---
 integration/test_servers_of_happiness.py | 2 +-
 integration/test_tor.py                  | 6 +++---
 integration/util.py                      | 4 ++--
 src/allmydata/test/cli_node_api.py       | 1 +
 src/allmydata/test/test_runner.py        | 2 +-
 src/allmydata/test/test_system.py        | 2 +-
 6 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index 97392bf00..1f350eb8e 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -30,7 +30,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         proto,
         sys.executable,
         [
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', node_dir,
             'put', __file__,
         ]
diff --git a/integration/test_tor.py b/integration/test_tor.py
index dcbfb1151..3b374f669 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -46,7 +46,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         proto,
         sys.executable,
         (
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'carol'),
             'put', gold_path,
         )
@@ -60,7 +60,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         proto,
         sys.executable,
         (
-            sys.executable, '-m', 'allmydata.scripts.runner',
+            sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'dave'),
             'get', cap,
         )
@@ -84,7 +84,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
             proto,
             sys.executable,
             (
-                sys.executable, '-m', 'allmydata.scripts.runner',
+                sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                 'create-node',
                 '--nickname', name,
                 '--introducer', introducer_furl,
diff --git a/integration/util.py b/integration/util.py
index 256fd68c1..b72e11c72 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -152,9 +152,9 @@ def _tahoe_runner_optional_coverage(proto, reactor, request, other_args):
     `--coverage` option if the `request` indicates we should.
     """
     if request.config.getoption('coverage'):
-        args = [sys.executable, '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
+        args = [sys.executable, '-b', '-m', 'coverage', 'run', '-m', 'allmydata.scripts.runner', '--coverage']
     else:
-        args = [sys.executable, '-m', 'allmydata.scripts.runner']
+        args = [sys.executable, '-b', '-m', 'allmydata.scripts.runner']
     args += other_args
     return reactor.spawnProcess(
         proto,
diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py
index 4e4173924..be0381e11 100644
--- a/src/allmydata/test/cli_node_api.py
+++ b/src/allmydata/test/cli_node_api.py
@@ -154,6 +154,7 @@ class CLINodeAPI(object):
         exe = sys.executable
         argv = [
             exe,
+            "-b",
             u"-m",
             u"allmydata.scripts.runner",
         ] + argv
diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index c80b5dc9c..7cc89c287 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -88,7 +88,7 @@ def run_bintahoe(extra_argv, python_options=None):
     argv = [executable]
     if python_options is not None:
         argv.extend(python_options)
-    argv.extend([u"-m", u"allmydata.scripts.runner"])
+    argv.extend([u"-b", u"-m", u"allmydata.scripts.runner"])
     argv.extend(extra_argv)
     argv = list(unicode_to_argv(arg) for arg in argv)
     p = Popen(argv, stdout=PIPE, stderr=PIPE)
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 040104b4c..0ff1e06e9 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -76,7 +76,7 @@ class RunBinTahoeMixin(object):
         # support env yet and is also synchronous.  If we could get rid of
         # this in favor of that, though, it would probably be an improvement.
         command = sys.executable
-        argv = python_options + ["-m", "allmydata.scripts.runner"] + args
+        argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args
 
         if env is None:
             env = os.environ

From 08772c5a86b3f3b360e17891e3d3bf887bb68462 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 11:58:55 -0400
Subject: [PATCH 13/81] News file.

---
 newsfragments/3619.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3619.minor

diff --git a/newsfragments/3619.minor b/newsfragments/3619.minor
new file mode 100644
index 000000000..e69de29bb

From abb247b3cc120568a9765c89d7f0ea61faea3072 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 16 Apr 2021 12:01:07 -0400
Subject: [PATCH 14/81] Fix flake.

---
 src/allmydata/test/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py
index 45536a6c6..abf23a301 100644
--- a/src/allmydata/test/__init__.py
+++ b/src/allmydata/test/__init__.py
@@ -20,7 +20,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 

From 1b46f981c6386c1d092d8752699f40eea96aa32d Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Fri, 9 Apr 2021 13:43:43 -0400
Subject: [PATCH 15/81] Remove AccountURLChecker, the code that relies on it,
 associated tests, and docs.

---
 docs/frontends/FTP-and-SFTP.rst  | 27 ----------------
 newsfragments/3652.removed       |  1 +
 src/allmydata/client.py          |  4 +--
 src/allmydata/frontends/auth.py  | 55 --------------------------------
 src/allmydata/frontends/sftpd.py | 11 +++----
 5 files changed, 6 insertions(+), 92 deletions(-)
 create mode 100644 newsfragments/3652.removed

diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst
index ee6371812..4c87b0bc4 100644
--- a/docs/frontends/FTP-and-SFTP.rst
+++ b/docs/frontends/FTP-and-SFTP.rst
@@ -78,33 +78,6 @@ start with "ssh-".
 Now add an ``accounts.file`` directive to your ``tahoe.cfg`` file, as described in
 the next sections.
 
-Running An Account Server (accounts.url)
-========================================
-
-The accounts.url directive allows access requests to be controlled by an
-HTTP-based login service, useful for centralized deployments. This was used
-by AllMyData to provide web-based file access, where the service used a
-simple PHP script and database lookups to map an account email address and
-password to a Tahoe-LAFS directory cap. The service will receive a
-multipart/form-data POST, just like one created with a <form> and <input>
-fields, with three parameters:
-
-• action: "authenticate" (this is a static string)
-• email: USERNAME (Tahoe-LAFS has no notion of email addresses, but the
-  authentication service uses them as account names, so the interface
-  presents this argument as "email" rather than "username").
-• passwd: PASSWORD
-
-It should return a single string that either contains a Tahoe-LAFS directory
-cap (URI:DIR2:...), or "0" to indicate a login failure.
-
-Tahoe-LAFS recommends the service be secure, preferably localhost-only.  This
-makes it harder for attackers to brute force the password or use DNS
-poisoning to cause the Tahoe-LAFS gateway to talk with the wrong server,
-thereby revealing the usernames and passwords.
-
-Public key authentication is not supported when an account server is used.
-
 Configuring SFTP Access
 =======================
 
diff --git a/newsfragments/3652.removed b/newsfragments/3652.removed
new file mode 100644
index 000000000..a3e964702
--- /dev/null
+++ b/newsfragments/3652.removed
@@ -0,0 +1 @@
+Removed support for the Account Server frontend authentication type.
diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 3bf976fe5..a6c45643f 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -116,7 +116,6 @@ _client_config = configutil.ValidConfiguration(
         ),
         "sftpd": (
             "accounts.file",
-            "accounts.url",
             "enabled",
             "host_privkey_file",
             "host_pubkey_file",
@@ -1042,13 +1041,12 @@ class _Client(node.Node, pollmixin.PollMixin):
             accountfile = self.config.get_config("sftpd", "accounts.file", None)
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
-            accounturl = self.config.get_config("sftpd", "accounts.url", None)
             sftp_portstr = self.config.get_config("sftpd", "port", "tcp:8022")
             pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
             privkey_file = self.config.get_config("sftpd", "host_privkey_file")
 
             from allmydata.frontends import sftpd
-            s = sftpd.SFTPServer(self, accountfile, accounturl,
+            s = sftpd.SFTPServer(self, accountfile,
                                  sftp_portstr, pubkey_file, privkey_file)
             s.setServiceParent(self)
 
diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py
index 7f81572fe..f2ac99b8f 100644
--- a/src/allmydata/frontends/auth.py
+++ b/src/allmydata/frontends/auth.py
@@ -1,14 +1,10 @@
-import os
-
 from zope.interface import implementer
-from twisted.web.client import getPage
 from twisted.internet import defer
 from twisted.cred import error, checkers, credentials
 from twisted.conch.ssh import keys
 from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB
 
 from allmydata.util.dictutil import BytesKeyDict
-from allmydata.util import base32
 from allmydata.util.fileutil import abspath_expanduser_unicode
 
 
@@ -86,54 +82,3 @@ class AccountFileChecker(object):
         d = defer.maybeDeferred(creds.checkPassword, correct)
         d.addCallback(self._cbPasswordMatch, str(creds.username))
         return d
-
-
-@implementer(checkers.ICredentialsChecker)
-class AccountURLChecker(object):
-    credentialInterfaces = (credentials.IUsernamePassword,)
-
-    def __init__(self, client, auth_url):
-        self.client = client
-        self.auth_url = auth_url
-
-    def _cbPasswordMatch(self, rootcap, username):
-        return FTPAvatarID(username, rootcap)
-
-    def post_form(self, username, password):
-        sepbase = base32.b2a(os.urandom(4))
-        sep = "--" + sepbase
-        form = []
-        form.append(sep)
-        fields = {"action": "authenticate",
-                  "email": username,
-                  "passwd": password,
-                  }
-        for name, value in fields.iteritems():
-            form.append('Content-Disposition: form-data; name="%s"' % name)
-            form.append('')
-            assert isinstance(value, str)
-            form.append(value)
-            form.append(sep)
-        form[-1] += "--"
-        body = "\r\n".join(form) + "\r\n"
-        headers = {"content-type": "multipart/form-data; boundary=%s" % sepbase,
-                   }
-        return getPage(self.auth_url, method="POST",
-                       postdata=body, headers=headers,
-                       followRedirect=True, timeout=30)
-
-    def _parse_response(self, res):
-        rootcap = res.strip()
-        if rootcap == "0":
-            raise error.UnauthorizedLogin
-        return rootcap
-
-    def requestAvatarId(self, credentials):
-        # construct a POST to the login form. While this could theoretically
-        # be done with something like the stdlib 'email' package, I can't
-        # figure out how, so we just slam together a form manually.
-        d = self.post_form(credentials.username, credentials.password)
-        d.addCallback(self._parse_response)
-        d.addCallback(self._cbPasswordMatch, str(credentials.username))
-        return d
-
diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py
index bc7196de6..17eca993e 100644
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@@ -1983,7 +1983,7 @@ class ShellSession(PrefixingLogMixin):
 components.registerAdapter(ShellSession, SFTPUserHandler, ISession)
 
 
-from allmydata.frontends.auth import AccountURLChecker, AccountFileChecker, NeedRootcapLookupScheme
+from allmydata.frontends.auth import AccountFileChecker, NeedRootcapLookupScheme
 
 @implementer(portal.IRealm)
 class Dispatcher(object):
@@ -2000,7 +2000,7 @@ class Dispatcher(object):
 class SFTPServer(service.MultiService):
     name = "frontend:sftp"
 
-    def __init__(self, client, accountfile, accounturl,
+    def __init__(self, client, accountfile,
                  sftp_portstr, pubkey_file, privkey_file):
         precondition(isinstance(accountfile, (str, type(None))), accountfile)
         precondition(isinstance(pubkey_file, str), pubkey_file)
@@ -2013,12 +2013,9 @@ class SFTPServer(service.MultiService):
         if accountfile:
             c = AccountFileChecker(self, accountfile)
             p.registerChecker(c)
-        if accounturl:
-            c = AccountURLChecker(self, accounturl)
-            p.registerChecker(c)
-        if not accountfile and not accounturl:
+        if not accountfile:
             # we could leave this anonymous, with just the /uri/CAP form
-            raise NeedRootcapLookupScheme("must provide an account file or URL")
+            raise NeedRootcapLookupScheme("must provide an account file")
 
         pubkey = keys.Key.fromFile(pubkey_file.encode(get_filesystem_encoding()))
         privkey = keys.Key.fromFile(privkey_file.encode(get_filesystem_encoding()))

From 72a370992fbc41837a1d88a11f3fae8c5d2ab6d1 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 17 Apr 2021 18:24:22 -0400
Subject: [PATCH 16/81] Remove ToC entry for account server.

---
 docs/frontends/FTP-and-SFTP.rst | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/frontends/FTP-and-SFTP.rst b/docs/frontends/FTP-and-SFTP.rst
index 4c87b0bc4..9d4f1dcec 100644
--- a/docs/frontends/FTP-and-SFTP.rst
+++ b/docs/frontends/FTP-and-SFTP.rst
@@ -7,11 +7,10 @@ Tahoe-LAFS SFTP Frontend
 1.  `SFTP Background`_
 2.  `Tahoe-LAFS Support`_
 3.  `Creating an Account File`_
-4.  `Running An Account Server (accounts.url)`_
-5.  `Configuring SFTP Access`_
-6.  `Dependencies`_
-7.  `Immutable and Mutable Files`_
-8.  `Known Issues`_
+4.  `Configuring SFTP Access`_
+5.  `Dependencies`_
+6.  `Immutable and Mutable Files`_
+7.  `Known Issues`_
 
 
 SFTP Background

From 6e8dde3b14b80f6bdccd6c6ba263d315d9eed7bc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 10:09:03 -0400
Subject: [PATCH 17/81] Simplify.

---
 src/allmydata/util/jsonbytes.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index 995165ee6..2d3cb7504 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -27,16 +27,18 @@ if PY2:
 
     codecs.register_error("backslashreplace_tahoe_py2", backslashreplace_py2)
 
-def _make_bytes_to_unicode(any_bytes):
+
+def _bytes_to_unicode(any_bytes, obj):
     """Create a function that recursively converts bytes to unicode.
 
     :param any_bytes: If True, also support non-UTF-8-encoded bytes.
+    :param obj: Object to de-byte-ify.
     """
     errors = "backslashreplace" if any_bytes else "strict"
     if PY2 and errors == "backslashreplace":
         errors = "backslashreplace_tahoe_py2"
 
-    def _bytes_to_unicode(obj):
+    def doit(obj):
         """Convert any bytes objects to unicode, recursively."""
         if isinstance(obj, bytes):
             return obj.decode("utf-8", errors=errors)
@@ -45,14 +47,14 @@ def _make_bytes_to_unicode(any_bytes):
             for k, v in obj.items():
                 if isinstance(k, bytes):
                     k = k.decode("utf-8", errors=errors)
-                v = _bytes_to_unicode(v)
+                v = doit(v)
                 new_obj[k] = v
             return new_obj
         if isinstance(obj, (list, set, tuple)):
-            return [_bytes_to_unicode(i) for i in obj]
+            return [doit(i) for i in obj]
         return obj
 
-    return _bytes_to_unicode
+    return doit(obj)
 
 
 class UTF8BytesJSONEncoder(json.JSONEncoder):
@@ -61,7 +63,7 @@ class UTF8BytesJSONEncoder(json.JSONEncoder):
     """
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
-            self, _make_bytes_to_unicode(False)(o), **kwargs)
+            self, _bytes_to_unicode(False, o), **kwargs)
 
 
 class AnyBytesJSONEncoder(json.JSONEncoder):
@@ -73,7 +75,7 @@ class AnyBytesJSONEncoder(json.JSONEncoder):
     """
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
-            self, _make_bytes_to_unicode(True)(o), **kwargs)
+            self, _bytes_to_unicode(True, o), **kwargs)
 
 
 def dumps(obj, *args, **kwargs):

From 08cb514eeea1226fca1c642207e70b19d3adf39a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 10:09:26 -0400
Subject: [PATCH 18/81] Correct parameter name.

---
 src/allmydata/util/jsonbytes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index 2d3cb7504..152b79861 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -96,7 +96,7 @@ def dumps(obj, *args, **kwargs):
 def dumps_bytes(obj, *args, **kwargs):
     """Encode to JSON, then encode as bytes.
 
-    :param bool all_bytes: If False (the default) the bytes are assumed to be
+    :param bool any_bytes: If False (the default) the bytes are assumed to be
         UTF-8 encoded Unicode strings.  If True, non-UTF-8 bytes are quoted for
         human consumption.
     """

From e090891935a13a4953ced575b60a5189c05808dc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 10:39:27 -0400
Subject: [PATCH 19/81] In PyPy encode() doesn't call iterencode().

---
 src/allmydata/util/jsonbytes.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index 152b79861..f6143f4d1 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -61,6 +61,10 @@ class UTF8BytesJSONEncoder(json.JSONEncoder):
     """
     A JSON encoder than can also encode UTF-8 encoded strings.
     """
+    def encode(self, o, **kwargs):
+        return json.JSONEncoder.encode(
+            self, _bytes_to_unicode(False, o), **kwargs)
+
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
             self, _bytes_to_unicode(False, o), **kwargs)
@@ -73,6 +77,10 @@ class AnyBytesJSONEncoder(json.JSONEncoder):
     Bytes are decoded to strings using UTF-8, if that fails to decode then the
     bytes are quoted.
     """
+    def encode(self, o, **kwargs):
+        return json.JSONEncoder.encode(
+            self, _bytes_to_unicode(True, o), **kwargs)
+
     def iterencode(self, o, **kwargs):
         return json.JSONEncoder.iterencode(
             self, _bytes_to_unicode(True, o), **kwargs)

From 83e16d40a452ac5766d07094243c497f37a3a624 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 11:18:36 -0400
Subject: [PATCH 20/81] Some tests passing on Python 3.

---
 src/allmydata/scripts/cli.py         |  6 +++---
 src/allmydata/scripts/tahoe_check.py |  8 ++++----
 src/allmydata/test/cli/test_check.py | 16 ++++++++--------
 3 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py
index 811ae7ef9..826c36e1f 100644
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions):
     def parseArgs(self, *args):
         if len(args) < 2:
             raise usage.UsageError("cp requires at least two arguments")
-        self.sources = map(argv_to_unicode, args[:-1])
+        self.sources = list(map(argv_to_unicode, args[:-1]))
         self.destination = argv_to_unicode(args[-1])
 
     synopsis = "[options] FROM.. TO"
@@ -435,7 +435,7 @@ class CheckOptions(FileStoreOptions):
         ("add-lease", None, "Add/renew lease on all shares."),
         ]
     def parseArgs(self, *locations):
-        self.locations = map(argv_to_unicode, locations)
+        self.locations = list(map(argv_to_unicode, locations))
 
     synopsis = "[options] [ALIAS:PATH]"
     description = """
@@ -452,7 +452,7 @@ class DeepCheckOptions(FileStoreOptions):
         ("verbose", "v", "Be noisy about what is happening."),
         ]
     def parseArgs(self, *locations):
-        self.locations = map(argv_to_unicode, locations)
+        self.locations = list(map(argv_to_unicode, locations))
 
     synopsis = "[options] [ALIAS:PATH]"
     description = """
diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index cef9e32be..1700e2b77 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -1,6 +1,6 @@
 from __future__ import print_function
 
-import urllib
+from urllib.parse import quote as url_quote
 import json
 
 # Python 2 compatibility
@@ -36,7 +36,7 @@ def check_location(options, where):
         return 1
     if path == '/':
         path = ''
-    url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+    url = nodeurl + "uri/%s" % url_quote(rootcap)
     if path:
         url += "/" + escape_path(path)
     # todo: should it end with a slash?
@@ -139,7 +139,7 @@ class DeepCheckOutput(LineOnlyReceiver, object):
         if self.in_error:
             print(quote_output(line, quotemarks=False), file=self.stderr)
             return
-        if line.startswith("ERROR:"):
+        if line.startswith(b"ERROR:"):
             self.in_error = True
             self.streamer.rc = 1
             print(quote_output(line, quotemarks=False), file=self.stderr)
@@ -297,7 +297,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object):
             return 1
         if path == '/':
             path = ''
-        url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+        url = nodeurl + "uri/%s" % url_quote(rootcap)
         if path:
             url += "/" + escape_path(path)
         # todo: should it end with a slash?
diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index 8cf963da6..eef8d3108 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -18,7 +18,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Check/check"
         self.set_up_grid()
         c0 = self.g.clients[0]
-        DATA = "data" * 100
+        DATA = b"data" * 100
         DATA_uploadable = MutableData(DATA)
         d = c0.create_mutable_file(DATA_uploadable)
         def _stash_uri(n):
@@ -45,7 +45,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(data["results"]["healthy"], True)
         d.addCallback(_check2)
 
-        d.addCallback(lambda ign: c0.upload(upload.Data("literal", convergence="")))
+        d.addCallback(lambda ign: c0.upload(upload.Data(b"literal", convergence=b"")))
         def _stash_lit_uri(n):
             self.lit_uri = n.get_uri()
         d.addCallback(_stash_lit_uri)
@@ -156,14 +156,14 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         c0 = self.g.clients[0]
         self.uris = {}
         self.fileurls = {}
-        DATA = "data" * 100
+        DATA = b"data" * 100
         quoted_good = quote_output(u"g\u00F6\u00F6d")
 
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
             self.rootnode = n
             self.rooturi = n.get_uri()
-            return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=""))
+            return n.add_file(u"g\u00F6\u00F6d", upload.Data(DATA, convergence=b""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_uri(fn, which):
             self.uris[which] = fn.get_uri()
@@ -171,11 +171,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_stash_uri, u"g\u00F6\u00F6d")
         d.addCallback(lambda ign:
                       self.rootnode.add_file(u"small",
-                                           upload.Data("literal",
-                                                        convergence="")))
+                                           upload.Data(b"literal",
+                                                        convergence=b"")))
         d.addCallback(_stash_uri, "small")
         d.addCallback(lambda ign:
-            c0.create_mutable_file(MutableData(DATA+"1")))
+            c0.create_mutable_file(MutableData(DATA+b"1")))
         d.addCallback(lambda fn: self.rootnode.set_node(u"mutable", fn))
         d.addCallback(_stash_uri, "mutable")
 
@@ -322,7 +322,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"subdir"))
         d.addCallback(_stash_uri, "subdir")
         d.addCallback(lambda fn:
-                      fn.add_file(u"subfile", upload.Data(DATA+"2", "")))
+                      fn.add_file(u"subfile", upload.Data(DATA+b"2", b"")))
         d.addCallback(lambda ign:
                       self.delete_shares_numbered(self.uris["subdir"],
                                                   range(10)))

From 5e59b9d8d6a059fd7f14129d7bf8785aa39a4fed Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 11:30:37 -0400
Subject: [PATCH 21/81] A little closer to passing tests on Python 3.

---
 src/allmydata/scripts/slow_operation.py |  4 ++--
 src/allmydata/scripts/tahoe_check.py    | 10 +++++++---
 src/allmydata/test/cli/test_check.py    | 15 ++++++++-------
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py
index ce25e9667..cce7d91c1 100644
--- a/src/allmydata/scripts/slow_operation.py
+++ b/src/allmydata/scripts/slow_operation.py
@@ -6,7 +6,7 @@ from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
 from allmydata.scripts.common_http import do_http, format_http_error
 from allmydata.util import base32
 from allmydata.util.encodingutil import quote_output, is_printable_ascii
-import urllib
+from urllib.parse import quote as url_quote
 import json
 
 class SlowOperationRunner(object):
@@ -27,7 +27,7 @@ class SlowOperationRunner(object):
             return 1
         if path == '/':
             path = ''
-        url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+        url = nodeurl + "uri/%s" % url_quote(rootcap)
         if path:
             url += "/" + escape_path(path)
         # todo: should it end with a slash?
diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index 1700e2b77..08569ec5d 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote
 import json
 
 # Python 2 compatibility
-from future.utils import PY2
+from future.utils import PY2, PY3
 if PY2:
     from future.builtins import str  # noqa: F401
 
@@ -54,8 +54,12 @@ def check_location(options, where):
         return 1
     jdata = resp.read()
     if options.get("raw"):
-        stdout.write(jdata)
-        stdout.write("\n")
+        if PY3:
+            stdoutb = stdout.buffer
+        else:
+            stdoutb = stdout
+        stdoutb.write(jdata)
+        stdoutb.write(b"\n")
         return 0
     data = json.loads(jdata)
 
diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index eef8d3108..e1a45faa2 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -1,3 +1,4 @@
+from past.builtins import unicode
 import os.path
 import json
 from twisted.trial import unittest
@@ -41,7 +42,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
             data = json.loads(out)
-            self.failUnlessReallyEqual(to_bytes(data["summary"]), "Healthy")
+            self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy")
             self.failUnlessReallyEqual(data["results"]["healthy"], True)
         d.addCallback(_check2)
 
@@ -68,7 +69,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(data["results"]["healthy"], True)
         d.addCallback(_check_lit_raw)
 
-        d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=""))
+        d.addCallback(lambda ign: c0.create_immutable_dirnode({}, convergence=b""))
         def _stash_lit_dir_uri(n):
             self.lit_dir_uri = n.get_uri()
         d.addCallback(_stash_lit_dir_uri)
@@ -89,9 +90,9 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             cso.parseOptions([shares[1][2]])
             storage_index = uri.from_string(self.uri).get_storage_index()
             self._corrupt_share_line = "  server %s, SI %s, shnum %d" % \
-                                       (base32.b2a(shares[1][1]),
-                                        base32.b2a(storage_index),
-                                        shares[1][0])
+                (unicode(base32.b2a(shares[1][1]), "ascii"),
+                 unicode(base32.b2a(storage_index), "ascii"),
+                 shares[1][0])
             debug.corrupt_share(cso)
         d.addCallback(_clobber_shares)
 
@@ -418,8 +419,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 0)
             self.failUnlessReallyEqual(err, "")
             #Ensure healthy appears for each uri
-            self.failUnlessIn("Healthy", out[:len(out)/2])
-            self.failUnlessIn("Healthy", out[len(out)/2:])
+            self.failUnlessIn("Healthy", out[:len(out)//2])
+            self.failUnlessIn("Healthy", out[len(out)//2:])
         d.addCallback(_check)
 
         d.addCallback(lambda ign: self.do_cli("check", self.uriList[0], "nonexistent:"))

From f6e0611b07c0e939fa96a27fa3d08a110b856a69 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 11:42:05 -0400
Subject: [PATCH 22/81] All tests pass on Python 3.

---
 src/allmydata/scripts/slow_operation.py | 3 ++-
 src/allmydata/scripts/tahoe_check.py    | 9 +++++++--
 src/allmydata/scripts/tahoe_manifest.py | 7 ++++---
 src/allmydata/test/cli/test_check.py    | 4 ++--
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py
index cce7d91c1..f7366fcc0 100644
--- a/src/allmydata/scripts/slow_operation.py
+++ b/src/allmydata/scripts/slow_operation.py
@@ -1,4 +1,5 @@
 from __future__ import print_function
+from six import ensure_str
 
 import os, time
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
@@ -14,7 +15,7 @@ class SlowOperationRunner(object):
     def run(self, options):
         stderr = options.stderr
         self.options = options
-        self.ophandle = ophandle = base32.b2a(os.urandom(16))
+        self.ophandle = ophandle = ensure_str(base32.b2a(os.urandom(16)))
         nodeurl = options['node-url']
         if not nodeurl.endswith("/"):
             nodeurl += "/"
diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index 08569ec5d..3859f0061 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -206,7 +206,7 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver, object):
         if self.in_error:
             print(quote_output(line, quotemarks=False), file=self.stderr)
             return
-        if line.startswith("ERROR:"):
+        if line.startswith(b"ERROR:"):
             self.in_error = True
             self.streamer.rc = 1
             print(quote_output(line, quotemarks=False), file=self.stderr)
@@ -321,12 +321,17 @@ class DeepCheckStreamer(LineOnlyReceiver, object):
             return 1
 
         # use Twisted to split this into lines
+        if PY3:
+            stdoutb = stdout.buffer
+        else:
+            stdoutb = stdout
+
         while True:
             chunk = resp.read(100)
             if not chunk:
                 break
             if self.options["raw"]:
-                stdout.write(chunk)
+                stdoutb.write(chunk)
             else:
                 output.dataReceived(chunk)
         if not self.options["raw"]:
diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py
index 386cdd1ad..6166564d3 100644
--- a/src/allmydata/scripts/tahoe_manifest.py
+++ b/src/allmydata/scripts/tahoe_manifest.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 
-import urllib, json
+from urllib.parse import quote as url_quote
+import json
 from twisted.protocols.basic import LineOnlyReceiver
 from allmydata.util.abbreviate import abbreviate_space_both
 from allmydata.scripts.slow_operation import SlowOperationRunner
@@ -35,7 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object):
             return 1
         if path == '/':
             path = ''
-        url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+        url = nodeurl + "uri/%s" % url_quote(rootcap)
         if path:
             url += "/" + escape_path(path)
         # todo: should it end with a slash?
@@ -63,7 +64,7 @@ class ManifestStreamer(LineOnlyReceiver, object):
         if self.in_error:
             print(quote_output(line, quotemarks=False), file=stderr)
             return
-        if line.startswith("ERROR:"):
+        if line.startswith(b"ERROR:"):
             self.in_error = True
             self.rc = 1
             print(quote_output(line, quotemarks=False), file=stderr)
diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index e1a45faa2..756ed4f17 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -237,8 +237,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             cso.parseOptions([shares[1][2]])
             storage_index = uri.from_string(self.uris["mutable"]).get_storage_index()
             self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \
-                                       (base32.b2a(shares[1][1]),
-                                        base32.b2a(storage_index),
+                                       (unicode(base32.b2a(shares[1][1]), "ascii"),
+                                        unicode(base32.b2a(storage_index), "ascii"),
                                         shares[1][0])
             debug.corrupt_share(cso)
         d.addCallback(_clobber_shares)

From 87f1620ab0a6f7bce4da9fe9c6bb8bd02274df53 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 11:42:23 -0400
Subject: [PATCH 23/81] News file.

---
 newsfragments/3678.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3678.minor

diff --git a/newsfragments/3678.minor b/newsfragments/3678.minor
new file mode 100644
index 000000000..e69de29bb

From 5ebb385c10f801fd72dc2248e748c3dcb1a69c65 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 21 Apr 2021 11:58:48 -0400
Subject: [PATCH 24/81] Port to Python 3.

---
 src/allmydata/test/cli/test_check.py | 57 ++++++++++++++++------------
 src/allmydata/util/_python3.py       |  1 +
 2 files changed, 34 insertions(+), 24 deletions(-)

diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index 756ed4f17..fda8b4352 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -1,4 +1,12 @@
-from past.builtins import unicode
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 import json
 from twisted.trial import unittest
@@ -13,6 +21,7 @@ from allmydata.scripts import debug
 from ..no_network import GridTestMixin
 from .common import CLITestMixin
 
+
 class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
 
     def test_check(self):
@@ -29,7 +38,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", self.uri))
         def _check1(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("Summary: Healthy" in lines, out)
@@ -39,7 +48,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", "--raw", self.uri))
         def _check2(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             data = json.loads(out)
             self.failUnlessReallyEqual(to_bytes(data["summary"]), b"Healthy")
@@ -54,7 +63,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", self.lit_uri))
         def _check_lit(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("Summary: Healthy (LIT)" in lines, out)
@@ -63,7 +72,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", "--raw", self.lit_uri))
         def _check_lit_raw(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             data = json.loads(out)
             self.failUnlessReallyEqual(data["results"]["healthy"], True)
@@ -90,8 +99,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             cso.parseOptions([shares[1][2]])
             storage_index = uri.from_string(self.uri).get_storage_index()
             self._corrupt_share_line = "  server %s, SI %s, shnum %d" % \
-                (unicode(base32.b2a(shares[1][1]), "ascii"),
-                 unicode(base32.b2a(storage_index), "ascii"),
+                (str(base32.b2a(shares[1][1]), "ascii"),
+                 str(base32.b2a(storage_index), "ascii"),
                  shares[1][0])
             debug.corrupt_share(cso)
         d.addCallback(_clobber_shares)
@@ -99,7 +108,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", "--verify", self.uri))
         def _check3(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             summary = [l for l in lines if l.startswith("Summary")][0]
@@ -113,7 +122,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("check", "--verify", "--raw", self.uri))
         def _check3_raw(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             data = json.loads(out)
             self.failUnlessReallyEqual(data["results"]["healthy"], False)
@@ -127,7 +136,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.do_cli("check", "--verify", "--repair", self.uri))
         def _check4(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("Summary: not healthy" in lines, out)
@@ -141,7 +150,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.do_cli("check", "--verify", "--repair", self.uri))
         def _check5(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("Summary: healthy" in lines, out)
@@ -183,7 +192,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
         def _check1(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("done: 4 objects checked, 4 healthy, 0 unhealthy"
@@ -199,7 +208,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                                               self.rooturi))
         def _check2(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
@@ -213,7 +222,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("stats", self.rooturi))
         def _check_stats(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnlessIn(" count-immutable-files: 1", lines)
@@ -237,8 +246,8 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             cso.parseOptions([shares[1][2]])
             storage_index = uri.from_string(self.uris["mutable"]).get_storage_index()
             self._corrupt_share_line = " corrupt: server %s, SI %s, shnum %d" % \
-                                       (unicode(base32.b2a(shares[1][1]), "ascii"),
-                                        unicode(base32.b2a(storage_index), "ascii"),
+                                       (str(base32.b2a(shares[1][1]), "ascii"),
+                                        str(base32.b2a(storage_index), "ascii"),
                                         shares[1][0])
             debug.corrupt_share(cso)
         d.addCallback(_clobber_shares)
@@ -252,7 +261,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.do_cli("deep-check", "--verbose", self.rooturi))
         def _check3(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
@@ -269,7 +278,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                                   self.rooturi))
         def _check4(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
@@ -288,7 +297,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                                   self.rooturi))
         def _check5(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             units = [json.loads(line) for line in lines]
@@ -302,7 +311,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                                   self.rooturi))
         def _check6(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnless("'<root>': healthy" in lines, out)
@@ -326,7 +335,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                       fn.add_file(u"subfile", upload.Data(DATA+b"2", b"")))
         d.addCallback(lambda ign:
                       self.delete_shares_numbered(self.uris["subdir"],
-                                                  range(10)))
+                                                  list(range(10))))
 
         # root
         # rootg\u00F6\u00F6d/
@@ -380,7 +389,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         d.addCallback(lambda ign: self.do_cli("deep-check"))
         d.addCallback(_check)
@@ -397,7 +406,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("nonexistent", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         return d
 
@@ -417,7 +426,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             #Ensure healthy appears for each uri
             self.failUnlessIn("Healthy", out[:len(out)//2])
             self.failUnlessIn("Healthy", out[len(out)//2:])
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 3003d2909..26f6b8e2a 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -176,6 +176,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_alias",
     "allmydata.test.cli.test_backup",
     "allmydata.test.cli.test_backupdb",
+    "allmydata.test.cli.test_check",
     "allmydata.test.cli.test_create",
     "allmydata.test.cli.test_invite",
     "allmydata.test.cli.test_status",

From 567c0f019e0a20f3aeb16de46b15ac820c0d3afe Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 09:39:50 -0400
Subject: [PATCH 25/81] Test random bytes.

---
 src/allmydata/test/web/test_logs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py
index 579e51dbc..89ec7ba42 100644
--- a/src/allmydata/test/web/test_logs.py
+++ b/src/allmydata/test/web/test_logs.py
@@ -109,7 +109,7 @@ class TestStreamingLogs(unittest.TestCase):
         def do_a_thing(arguments):
             pass
 
-        do_a_thing(arguments=[u"hello", b"good-day", 123, {"a": 35}, [None]])
+        do_a_thing(arguments=[u"hello", b"good-\xff-day", 123, {"a": 35}, [None]])
 
         proto.transport.loseConnection()
         yield proto.is_closed
@@ -117,7 +117,7 @@ class TestStreamingLogs(unittest.TestCase):
         self.assertEqual(len(messages), 2)
         self.assertEqual(messages[0]["action_type"], "test:cli:some-exciting-action")
         self.assertEqual(messages[0]["arguments"],
-                         ["hello", "good-day", 123, {"a": 35}, [None]])
+                         ["hello", "good-\\xff-day", 123, {"a": 35}, [None]])
         self.assertEqual(messages[1]["action_type"], "test:cli:some-exciting-action")
         self.assertEqual("started", messages[0]["action_status"])
         self.assertEqual("succeeded", messages[1]["action_status"])

From 86fe350bef7a340d28f5947b289de6a2a927b2fc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:15:43 -0400
Subject: [PATCH 26/81] Tests pass on Python 2.

---
 src/allmydata/test/cli/test_check.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index fda8b4352..320106be9 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_text
 
 import os.path
 import json
@@ -167,7 +168,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.uris = {}
         self.fileurls = {}
         DATA = b"data" * 100
-        quoted_good = quote_output(u"g\u00F6\u00F6d")
+        quoted_good = u"'g\u00F6\u00F6d'"
 
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
@@ -210,6 +211,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
+            out = ensure_text(out)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
             self.failUnless("'small': Healthy (LIT)" in lines, out)
@@ -263,6 +265,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
+            out = ensure_text(out)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
             self.failUnless("'small': Healthy (LIT)" in lines, out)
@@ -280,6 +283,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
+            out = ensure_text(out)
             lines = out.splitlines()
             self.failUnless("'<root>': Healthy" in lines, out)
             self.failUnless("'small': Healthy (LIT)" in lines, out)
@@ -313,6 +317,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(rc, 0)
+            out = ensure_text(out)
             lines = out.splitlines()
             self.failUnless("'<root>': healthy" in lines, out)
             self.failUnless("'small': healthy" in lines, out)
@@ -350,7 +355,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failIfEqual(rc, 0)
             self.failUnlessIn("ERROR: UnrecoverableFileError", err)
             # the fatal directory should still show up, as the last line
-            self.failUnlessIn(" subdir\n", out)
+            self.failUnlessIn(" subdir\n", ensure_text(out))
         d.addCallback(_manifest_failed)
 
         d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))

From 416813578a99ab5dc4aa1cb43e00e6cf455f8ce8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:18:59 -0400
Subject: [PATCH 27/81] Some progress towards passing tests on Python 3.

---
 src/allmydata/scripts/tahoe_cp.py | 32 ++++++++++++++++---------------
 src/allmydata/test/cli/test_cp.py |  2 ++
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py
index f7879f35c..58492a7a1 100644
--- a/src/allmydata/scripts/tahoe_cp.py
+++ b/src/allmydata/scripts/tahoe_cp.py
@@ -1,7 +1,9 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 import os.path
-import urllib
+from urllib.parse import quote as url_quote
 import json
 from collections import defaultdict
 from six.moves import cStringIO as StringIO
@@ -61,8 +63,8 @@ def mkdir(targeturl):
 
 def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
     url = nodeurl + "/".join(["uri",
-                              urllib.quote(parent_writecap),
-                              urllib.quote(unicode_to_url(name)),
+                              url_quote(parent_writecap),
+                              url_quote(unicode_to_url(name)),
                               ]) + "?t=mkdir"
     resp = do_http("POST", url)
     if resp.status in (200, 201):
@@ -199,7 +201,7 @@ class TahoeFileSource(object):
     def open(self, caps_only):
         if caps_only:
             return StringIO(self.readcap)
-        url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
+        url = self.nodeurl + "uri/" + url_quote(self.readcap)
         return GET_to_file(url)
 
     def bestcap(self):
@@ -239,7 +241,7 @@ class TahoeDirectorySource(object):
         self.writecap = writecap
         self.readcap = readcap
         bestcap = writecap or readcap
-        url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
+        url = self.nodeurl + "uri/%s" % url_quote(bestcap)
         resp = do_http("GET", url + "?t=json")
         if resp.status != 200:
             raise HTTPError("Error examining source directory", resp)
@@ -249,7 +251,7 @@ class TahoeDirectorySource(object):
         self.mutable = d.get("mutable", False) # older nodes don't provide it
         self.children_d = dict( [(unicode(name),value)
                                  for (name,value)
-                                 in d["children"].iteritems()] )
+                                 in d["children"].items()] )
         self.children = None
 
     def init_from_parsed(self, parsed):
@@ -259,7 +261,7 @@ class TahoeDirectorySource(object):
         self.mutable = d.get("mutable", False) # older nodes don't provide it
         self.children_d = dict( [(unicode(name),value)
                                  for (name,value)
-                                 in d["children"].iteritems()] )
+                                 in d["children"].items()] )
         self.children = None
 
     def populate(self, recurse):
@@ -329,14 +331,14 @@ class TahoeDirectoryTarget(object):
         self.mutable = d.get("mutable", False) # older nodes don't provide it
         self.children_d = dict( [(unicode(name),value)
                                  for (name,value)
-                                 in d["children"].iteritems()] )
+                                 in d["children"].items()] )
         self.children = None
 
     def init_from_grid(self, writecap, readcap):
         self.writecap = writecap
         self.readcap = readcap
         bestcap = writecap or readcap
-        url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
+        url = self.nodeurl + "uri/%s" % url_quote(bestcap)
         resp = do_http("GET", url + "?t=json")
         if resp.status != 200:
             raise HTTPError("Error examining target directory", resp)
@@ -346,7 +348,7 @@ class TahoeDirectoryTarget(object):
         self.mutable = d.get("mutable", False) # older nodes don't provide it
         self.children_d = dict( [(unicode(name),value)
                                  for (name,value)
-                                 in d["children"].iteritems()] )
+                                 in d["children"].items()] )
         self.children = None
 
     def just_created(self, writecap):
@@ -370,8 +372,8 @@ class TahoeDirectoryTarget(object):
                 url = None
                 if self.writecap:
                     url = self.nodeurl + "/".join(["uri",
-                                                   urllib.quote(self.writecap),
-                                                   urllib.quote(unicode_to_url(name))])
+                                                   url_quote(self.writecap),
+                                                   url_quote(unicode_to_url(name))])
                 self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
                                                       writecap, readcap, url)
             elif data[0] == "dirnode":
@@ -439,7 +441,7 @@ class TahoeDirectoryTarget(object):
     def set_children(self):
         if not self.new_children:
             return
-        url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
+        url = (self.nodeurl + "uri/" + url_quote(self.writecap)
                + "?t=set_children")
         set_data = {}
         for (name, filecap) in self.new_children.items():
@@ -603,7 +605,7 @@ class Copier(object):
                 t = LocalFileTarget(pathname) # non-empty
         else:
             # this is a tahoe object
-            url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
+            url = self.nodeurl + "uri/%s" % url_quote(rootcap)
             if path:
                 url += "/" + escape_path(path)
 
@@ -656,7 +658,7 @@ class Copier(object):
                 t = LocalFileSource(pathname, name) # non-empty
         else:
             # this is a tahoe object
-            url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
+            url = self.nodeurl + "uri/%s" % url_quote(rootcap)
             name = None
             if path:
                 if path.endswith("/"):
diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index 6cebec4a5..643a52bc6 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 import os.path, json
 from twisted.trial import unittest
 from twisted.python import usage

From 2aac69e0df288b4c1a55a6ef5ab62478075aa297 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:20:26 -0400
Subject: [PATCH 28/81] More passing tests on Python 3.

---
 src/allmydata/scripts/tahoe_cp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py
index 58492a7a1..0a6c67942 100644
--- a/src/allmydata/scripts/tahoe_cp.py
+++ b/src/allmydata/scripts/tahoe_cp.py
@@ -4,7 +4,6 @@ from past.builtins import unicode
 
 import os.path
 from urllib.parse import quote as url_quote
-import json
 from collections import defaultdict
 from six.moves import cStringIO as StringIO
 from twisted.python.failure import Failure
@@ -17,6 +16,7 @@ from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abs
 from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
     quote_local_unicode_path, to_bytes
 from allmydata.util.assertutil import precondition, _assert
+from allmydata.util import jsonbytes as json
 
 
 class MissingSourceError(TahoeError):
@@ -452,7 +452,7 @@ class TahoeDirectoryTarget(object):
             # TODO: think about how this affects forward-compatibility for
             # unknown caps
             set_data[name] = ["filenode", {"rw_uri": filecap}]
-        body = json.dumps(set_data)
+        body = json.dumps_bytes(set_data)
         POST(url, body)
 
 FileSources = (LocalFileSource, TahoeFileSource)

From b675ca23800fa52f7cb8ad2192952a0565587a2d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:23:58 -0400
Subject: [PATCH 29/81] Lint fix.

---
 src/allmydata/test/cli/test_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_check.py b/src/allmydata/test/cli/test_check.py
index 320106be9..e01dcc4cb 100644
--- a/src/allmydata/test/cli/test_check.py
+++ b/src/allmydata/test/cli/test_check.py
@@ -15,7 +15,7 @@ from six.moves import cStringIO as StringIO
 
 from allmydata import uri
 from allmydata.util import base32
-from allmydata.util.encodingutil import quote_output, to_bytes
+from allmydata.util.encodingutil import to_bytes
 from allmydata.mutable.publish import MutableData
 from allmydata.immutable import upload
 from allmydata.scripts import debug

From a393b54315e6f46bc82b2b3781125487b26215a2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:27:59 -0400
Subject: [PATCH 30/81] Fix BytesWarning errors.

---
 src/allmydata/scripts/slow_operation.py | 3 +++
 src/allmydata/scripts/tahoe_check.py    | 2 ++
 src/allmydata/scripts/tahoe_manifest.py | 3 +++
 3 files changed, 8 insertions(+)

diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py
index f7366fcc0..5d8c77538 100644
--- a/src/allmydata/scripts/slow_operation.py
+++ b/src/allmydata/scripts/slow_operation.py
@@ -1,4 +1,6 @@
 from __future__ import print_function
+
+from past.builtins import unicode
 from six import ensure_str
 
 import os, time
@@ -26,6 +28,7 @@ class SlowOperationRunner(object):
         except UnknownAliasError as e:
             e.display(stderr)
             return 1
+        path = unicode(path, "utf-8")
         if path == '/':
             path = ''
         url = nodeurl + "uri/%s" % url_quote(rootcap)
diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index 3859f0061..d8b7c9bce 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -34,6 +34,7 @@ def check_location(options, where):
     except UnknownAliasError as e:
         e.display(stderr)
         return 1
+    path = str(path, "utf-8")
     if path == '/':
         path = ''
     url = nodeurl + "uri/%s" % url_quote(rootcap)
@@ -299,6 +300,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object):
         except UnknownAliasError as e:
             e.display(stderr)
             return 1
+        path = str(path, "utf-8")
         if path == '/':
             path = ''
         url = nodeurl + "uri/%s" % url_quote(rootcap)
diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py
index 6166564d3..b837e648a 100644
--- a/src/allmydata/scripts/tahoe_manifest.py
+++ b/src/allmydata/scripts/tahoe_manifest.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 from urllib.parse import quote as url_quote
 import json
 from twisted.protocols.basic import LineOnlyReceiver
@@ -34,6 +36,7 @@ class ManifestStreamer(LineOnlyReceiver, object):
         except UnknownAliasError as e:
             e.display(stderr)
             return 1
+        path = unicode(path, "utf-8")
         if path == '/':
             path = ''
         url = nodeurl + "uri/%s" % url_quote(rootcap)

From 56e4385103828bf8033077c5309192a704042bf8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:36:10 -0400
Subject: [PATCH 31/81] More progress towards Python 3 tests passing.

---
 src/allmydata/scripts/tahoe_mkdir.py | 13 ++++++++-----
 src/allmydata/test/cli/test_cp.py    | 19 +++++--------------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_mkdir.py b/src/allmydata/scripts/tahoe_mkdir.py
index a76adc8fc..54e8ebe46 100644
--- a/src/allmydata/scripts/tahoe_mkdir.py
+++ b/src/allmydata/scripts/tahoe_mkdir.py
@@ -1,6 +1,8 @@
 from __future__ import print_function
 
-import urllib
+from past.builtins import unicode
+
+from urllib.parse import quote as url_quote
 from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError
 from allmydata.util.encodingutil import quote_output
@@ -24,7 +26,7 @@ def mkdir(options):
         # create a new unlinked directory
         url = nodeurl + "uri?t=mkdir"
         if options["format"]:
-            url += "&format=%s" % urllib.quote(options['format'])
+            url += "&format=%s" % url_quote(options['format'])
         resp = do_http("POST", url)
         rc = check_http_error(resp, stderr)
         if rc:
@@ -35,13 +37,14 @@ def mkdir(options):
         return 0
 
     # create a new directory at the given location
+    path = unicode(path, "utf-8")
     if path.endswith("/"):
         path = path[:-1]
     # path must be "/".join([s.encode("utf-8") for s in segments])
-    url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap),
-                                           urllib.quote(path))
+    url = nodeurl + "uri/%s/%s?t=mkdir" % (url_quote(rootcap),
+                                           url_quote(path))
     if options['format']:
-        url += "&format=%s" % urllib.quote(options['format'])
+        url += "&format=%s" % url_quote(options['format'])
 
     resp = do_http("POST", url)
     check_http_error(resp, stderr)
diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index 643a52bc6..fcd8c3bbf 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -26,12 +26,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
     def test_unicode_filename(self):
         self.basedir = "cli/Cp/unicode_filename"
 
-        fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
-        try:
-            fn1_arg = fn1.encode(get_io_encoding())
-            artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
+        fn1 = os.path.join(self.basedir, u"\u00C4rtonwall")
+        artonwall_arg = u"\u00C4rtonwall"
 
         skip_if_cannot_represent_filename(fn1)
 
@@ -46,7 +42,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         d = self.do_cli("create-alias", "tahoe")
 
-        d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:"))
+        d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:"))
 
         d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
         d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1))
@@ -202,13 +198,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
     def test_unicode_dirnames(self):
         self.basedir = "cli/Cp/unicode_dirnames"
 
-        fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
-        try:
-            fn1_arg = fn1.encode(get_io_encoding())
-            del fn1_arg # hush pyflakes
-            artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
+        fn1 = os.path.join(self.basedir, u"\u00C4rtonwall")
+        artonwall_arg = u"\u00C4rtonwall"
 
         skip_if_cannot_represent_filename(fn1)
 

From b85d735b8bbba53b12f198ea6fd99af60af9c263 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 22 Apr 2021 10:43:55 -0400
Subject: [PATCH 32/81] Even more progress towards Python 3 tests passing.

---
 src/allmydata/scripts/tahoe_put.py |  9 ++++++---
 src/allmydata/test/cli/test_cp.py  | 10 ++++++++--
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py
index 8d87408dc..b64283a81 100644
--- a/src/allmydata/scripts/tahoe_put.py
+++ b/src/allmydata/scripts/tahoe_put.py
@@ -1,7 +1,9 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 from six.moves import cStringIO as StringIO
-import urllib
+from urllib.parse import quote as url_quote
 
 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
@@ -46,19 +48,20 @@ def put(options):
 
         # FIXME: don't hardcode cap format.
         if to_file.startswith("URI:MDMF:") or to_file.startswith("URI:SSK:"):
-            url = nodeurl + "uri/%s" % urllib.quote(to_file)
+            url = nodeurl + "uri/%s" % url_quote(to_file)
         else:
             try:
                 rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS)
             except UnknownAliasError as e:
                 e.display(stderr)
                 return 1
+            path = unicode(path, "utf-8")
             if path.startswith("/"):
                 suggestion = to_file.replace(u"/", u"", 1)
                 print("Error: The remote filename must not start with a slash", file=stderr)
                 print("Please try again, perhaps with %s" % quote_output(suggestion), file=stderr)
                 return 1
-            url = nodeurl + "uri/%s/" % urllib.quote(rootcap)
+            url = nodeurl + "uri/%s/" % url_quote(rootcap)
             if path:
                 url += escape_path(path)
     else:
diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index fcd8c3bbf..e7c9d2323 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+from future.utils import PY2
 from past.builtins import unicode
 
 import os.path, json
@@ -64,7 +65,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 self.failUnlessIn("files whose names could not be converted", err)
             else:
                 self.failUnlessReallyEqual(rc, 0)
-                self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n")
+                if PY2:
+                    out = out.decode(get_io_encoding())
+                self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n")
                 self.failUnlessReallyEqual(err, "")
         d.addCallback(_check)
 
@@ -220,7 +223,9 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 self.failUnlessIn("files whose names could not be converted", err)
             else:
                 self.failUnlessReallyEqual(rc, 0)
-                self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n")
+                if PY2:
+                    out = out.decode(get_io_encoding())
+                self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n")
                 self.failUnlessReallyEqual(err, "")
         d.addCallback(_check)
 
@@ -259,6 +264,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _get_test_txt_uris(args):
             (rc, out, err) = args
             self.failUnlessEqual(rc, 0)
+            import pdb; pdb.set_trace()
             filetype, data = json.loads(out)
 
             self.failUnlessEqual(filetype, "filenode")

From eb5211672c8752c07a66e6f3450b0f23b8918122 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 26 Apr 2021 09:46:21 -0400
Subject: [PATCH 33/81] Tests pass on Python 3.

---
 src/allmydata/scripts/tahoe_ls.py | 2 +-
 src/allmydata/scripts/tahoe_mv.py | 6 +++---
 src/allmydata/test/cli/test_cp.py | 9 ++++-----
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py
index 91665e77b..71db3f5cb 100644
--- a/src/allmydata/scripts/tahoe_ls.py
+++ b/src/allmydata/scripts/tahoe_ls.py
@@ -45,10 +45,10 @@ def list(options):
             return resp.status
 
     data = resp.read()
-
     if options['json']:
         # The webapi server should always output printable ASCII.
         if is_printable_ascii(data):
+            data = unicode(data, "ascii")
             print(data, file=stdout)
             return 0
         else:
diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py
index 7d13ea72a..bdaf134ff 100644
--- a/src/allmydata/scripts/tahoe_mv.py
+++ b/src/allmydata/scripts/tahoe_mv.py
@@ -1,7 +1,7 @@
 from __future__ import print_function
 
 import re
-import urllib
+from urllib.parse import quote as url_quote
 import json
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
@@ -25,7 +25,7 @@ def mv(options, mode="move"):
     except UnknownAliasError as e:
         e.display(stderr)
         return 1
-    from_url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+    from_url = nodeurl + "uri/%s" % url_quote(rootcap)
     if from_path:
         from_url += "/" + escape_path(from_path)
     # figure out the source cap
@@ -43,7 +43,7 @@ def mv(options, mode="move"):
     except UnknownAliasError as e:
         e.display(stderr)
         return 1
-    to_url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+    to_url = nodeurl + "uri/%s" % url_quote(rootcap)
     if path:
         to_url += "/" + escape_path(path)
 
diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index e7c9d2323..a63718cae 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -99,7 +99,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         fn1 = os.path.join(self.basedir, "Metallica")
         fn2 = os.path.join(outdir, "Not Metallica")
         fn3 = os.path.join(outdir, "test2")
-        DATA1 = "puppies" * 10000
+        DATA1 = b"puppies" * 10000
         fileutil.write(fn1, DATA1)
 
         d = self.do_cli("create-alias", "tahoe")
@@ -264,7 +264,6 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _get_test_txt_uris(args):
             (rc, out, err) = args
             self.failUnlessEqual(rc, 0)
-            import pdb; pdb.set_trace()
             filetype, data = json.loads(out)
 
             self.failUnlessEqual(filetype, "filenode")
@@ -817,9 +816,9 @@ cp -r $DIRCAP5 $DIRCAP6 to : E9-COLLIDING-TARGETS
 """
 
 class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
-    FILE_CONTENTS = "file text"
-    FILE_CONTENTS_5 = "5"
-    FILE_CONTENTS_6 = "6"
+    FILE_CONTENTS = b"file text"
+    FILE_CONTENTS_5 = b"5"
+    FILE_CONTENTS_6 = b"6"
 
     def do_setup(self):
         # first we build a tahoe filesystem that contains:

From 04a09558b60978d1a94fe5a6a9277cd1d9ee1dbf Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 26 Apr 2021 09:54:35 -0400
Subject: [PATCH 34/81] Port to Python 3.

---
 src/allmydata/test/cli/test_cp.py | 22 +++++++++++++++-------
 src/allmydata/util/_python3.py    |  1 +
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index a63718cae..c46ba1f84 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -1,7 +1,15 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
 
 from future.utils import PY2
-from past.builtins import unicode
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_text
 
 import os.path, json
 from twisted.trial import unittest
@@ -46,12 +54,12 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:"))
 
         d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
-        d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1))
+        d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1))
 
         d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
 
         d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
-        d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
+        d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA2))
 
         d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
         def _check(args):
@@ -68,7 +76,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 if PY2:
                     out = out.decode(get_io_encoding())
                 self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n")
-                self.failUnlessReallyEqual(err, "")
+                self.assertEqual(len(err), 0, err)
         d.addCallback(_check)
 
         return d
@@ -129,7 +137,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("when copying into a directory, all source files must have names, but",
                               err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_resp)
 
         # Create a directory, linked at tahoe:test .
@@ -218,7 +226,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 unicode_to_output(u"\u00C4rtonwall")
             except UnicodeEncodeError:
                 self.failUnlessReallyEqual(rc, 1)
-                self.failUnlessReallyEqual(out, "")
+                self.assertEqual(len(out), 0, out)
                 self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
                 self.failUnlessIn("files whose names could not be converted", err)
             else:
@@ -226,7 +234,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
                 if PY2:
                     out = out.decode(get_io_encoding())
                 self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n")
-                self.failUnlessReallyEqual(err, "")
+                self.assertEqual(len(err), 0, err)
         d.addCallback(_check)
 
         return d
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 26f6b8e2a..471c7e913 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -177,6 +177,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_backup",
     "allmydata.test.cli.test_backupdb",
     "allmydata.test.cli.test_check",
+    "allmydata.test.cli.test_cp",
     "allmydata.test.cli.test_create",
     "allmydata.test.cli.test_invite",
     "allmydata.test.cli.test_status",

From f424e906ad3d7438ee540bd9ffc8e25c86bb5ae4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 26 Apr 2021 09:54:58 -0400
Subject: [PATCH 35/81] News file.

---
 newsfragments/3679.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3679.minor

diff --git a/newsfragments/3679.minor b/newsfragments/3679.minor
new file mode 100644
index 000000000..e69de29bb

From 2ad8a4745519eac032cee4d2f46189cc3fa117b3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 26 Apr 2021 09:59:18 -0400
Subject: [PATCH 36/81] Fix flake.

---
 src/allmydata/test/cli/test_cp.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_cp.py b/src/allmydata/test/cli/test_cp.py
index c46ba1f84..d198a832c 100644
--- a/src/allmydata/test/cli/test_cp.py
+++ b/src/allmydata/test/cli/test_cp.py
@@ -9,7 +9,6 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-from six import ensure_text
 
 import os.path, json
 from twisted.trial import unittest

From b707a6ca7b33eafd6644eff7ef8c3824a4a1b87f Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 11:36:51 -0400
Subject: [PATCH 37/81] GitHub Actions: do not install vcpython27

Microsoft seems to have pulled the compiler download.
---
 .github/workflows/ci.yml | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c45ceaa63..6bec373f8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,15 +27,6 @@ jobs:
 
     steps:
 
-      # Get vcpython27 on Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       # See https://github.com/actions/checkout. A fetch-depth of 0
       # fetches all tags and branches.
       - name: Check out Tahoe-LAFS sources
@@ -164,15 +155,6 @@ jobs:
 
     steps:
 
-      # Get vcpython27 for Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       - name: Install Tor [Ubuntu]
         if: matrix.os == 'ubuntu-latest'
         run: sudo apt install tor
@@ -242,15 +224,6 @@ jobs:
 
     steps:
 
-      # Get vcpython27 for Windows + Python 2.7, to build netifaces
-      # extension.  See https://chocolatey.org/packages/vcpython27 and
-      # https://github.com/crazy-max/ghaction-chocolatey
-      - name: Install MSVC 9.0 for Python 2.7 [Windows]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
-        uses: crazy-max/ghaction-chocolatey@v1
-        with:
-          args: install vcpython27
-
       - name: Check out Tahoe-LAFS sources
         uses: actions/checkout@v2
         with:

From 196ce5103dd203fd7cc59310fa55d3bd851a14d5 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 12:00:11 -0400
Subject: [PATCH 38/81] GitHub Actions: test with 32-bit Python 2.7 on Windows

---
 .github/workflows/ci.yml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6bec373f8..9f0515f86 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,10 +35,22 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # We use netifaces, which does not ship a 64-bit wheel for
+      # Windows, but it ships a 32-bit wheel.  Since MS has pulled
+      # vcpython27 compiler, building a netifaces wheel is not an
+      # option anymore, so let us test on 32-bit Windows.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
       # To use pip caching with GitHub Actions in an OS-independent
       # manner, we need `pip cache dir` command, which became
       # available since pip v20.1+.  At the time of writing this,
@@ -175,10 +187,19 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See this step under coverage job.          
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -230,10 +251,19 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
+        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See this step under coverage job.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x86]
+        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
       - name: Get pip cache directory
         id: pip-cache
         run: |

From f4b8780ba7c2ef136f60635c0c9fa56f41197f7c Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 12:00:37 -0400
Subject: [PATCH 39/81] Add newsfragment

---
 newsfragments/3681.installations | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 newsfragments/3681.installations

diff --git a/newsfragments/3681.installations b/newsfragments/3681.installations
new file mode 100644
index 000000000..a697e6c60
--- /dev/null
+++ b/newsfragments/3681.installations
@@ -0,0 +1,3 @@
+Tahoe-LAFS CI now runs tests only on 32-bit Windows. Microsoft has
+removed vcpython27 compiler downloads from their site, and Tahoe-LAFS
+needs vcpython27 to build and install netifaces on 64-bit Windows.

From 106976e8cc8d119025b1dc307bb9a722e26da87d Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 12:17:25 -0400
Subject: [PATCH 40/81] GitHub Actions: use expression syntax

Per GitHub documentation:

  When you use expressions in an if conditional, you may omit the
  expression syntax (${{ }}) because GitHub automatically evaluates
  the if conditional as an expression, unless the expression contains
  any operators. If the expression contains any operators, the
  expression must be contained within ${{ }} to explicitly mark it for
  evaluation.

https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions
---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9f0515f86..61e40fcd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
+        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -45,7 +45,7 @@ jobs:
       # vcpython27 compiler, building a netifaces wheel is not an
       # option anymore, so let us test on 32-bit Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -187,14 +187,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
+        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.          
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -251,14 +251,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: matrix.os != 'windows-latest' && matrix.python-version != '2.7'
+        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: matrix.os == 'windows-latest' && matrix.python-version == '2.7'
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From ed82119f32af602c21201e572a4b2c705d892038 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 13:14:28 -0400
Subject: [PATCH 41/81] GitHub Actions: when in doubt, throw in more parens

---
 .github/workflows/ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 61e40fcd0..b82f3cbda 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
+        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -45,7 +45,7 @@ jobs:
       # vcpython27 compiler, building a netifaces wheel is not an
       # option anymore, so let us test on 32-bit Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
+        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -187,14 +187,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
+        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
-      # See this step under coverage job.          
+      # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
+        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -251,14 +251,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ matrix.os != 'windows-latest' && matrix.python-version != '2.7' }}
+        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
+        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From dec97b3aa119e6f242459d2d188d4c44598778cb Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 13:35:37 -0400
Subject: [PATCH 42/81] GitHub Actions: when in doubt, drop some curly braces

---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b82f3cbda..2fe6bf9b0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
+        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -45,7 +45,7 @@ jobs:
       # vcpython27 compiler, building a netifaces wheel is not an
       # option anymore, so let us test on 32-bit Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
+        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -187,14 +187,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
+        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
+        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -251,14 +251,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ${{ ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' ) }}
+        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' ) }}
+        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From 68603fc015898a06d23052735b2059fb880323f6 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 13:41:17 -0400
Subject: [PATCH 43/81] GitHub Actions: just use x86 Python to test on Windows

I can't figure out the correct GitHub Actions magic incantation^w^w
expression syntax that is needed to isolate (Windows && Python 2.7),
so let's just run also run x86 Python 3.6 on Windows.
---
 .github/workflows/ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2fe6bf9b0..ab0ba3f08 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -45,7 +45,7 @@ jobs:
       # vcpython27 compiler, building a netifaces wheel is not an
       # option anymore, so let us test on 32-bit Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
+        if: ${{ matrix.os == 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -187,14 +187,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
+        if: ${{ matrix.os == 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -251,14 +251,14 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        if: ( matrix.os != 'windows-latest' && matrix.python-version != '2.7' )
+        if: ${{ matrix.os != 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
 
       # See this step under coverage job.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ( matrix.os == 'windows-latest' && matrix.python-version == '2.7' )
+        if: ${{ matrix.os == 'windows-latest' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From 1a05fb21de203c9b3c31e8a6143da12a6cd48157 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 14:02:26 -0400
Subject: [PATCH 44/81] Name newsfragment correctly

---
 newsfragments/{3681.installations => 3681.installation} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename newsfragments/{3681.installations => 3681.installation} (100%)

diff --git a/newsfragments/3681.installations b/newsfragments/3681.installation
similarity index 100%
rename from newsfragments/3681.installations
rename to newsfragments/3681.installation

From e46e4409c2649dc4549ab256d3dee618fd99dbf7 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 16:39:23 -0400
Subject: [PATCH 45/81] GitHub Actions: really use x86 Python on Windows

In a prior commit, I mistakenly used "architecture: x64" instead of
"architecture: x86", and tests actually passed.

That was surprising, because netifaces do not have amd64_win will on
PyPI.  But mystery was solved when itamarst pointed out that netifaces
wheel (that we previously built) is present on pip cache.

But pip cache might be purged one day, and tests will fail again that
day.  We can't have that, so we will try to stick with x86 for now.
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ab0ba3f08..bfa705d28 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,7 +49,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: x86
 
       # To use pip caching with GitHub Actions in an OS-independent
       # manner, we need `pip cache dir` command, which became
@@ -198,7 +198,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: x86
 
       - name: Get pip cache directory
         id: pip-cache
@@ -262,7 +262,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: x86
 
       - name: Get pip cache directory
         id: pip-cache

From 1531bea63fda3851ed988ac2b97574c413741cbb Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 16:58:24 -0400
Subject: [PATCH 46/81] GitHub Actions: update note about Windows

[skip ci]
---
 .github/workflows/ci.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bfa705d28..2adb22bc8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -40,10 +40,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # We use netifaces, which does not ship a 64-bit wheel for
-      # Windows, but it ships a 32-bit wheel.  Since MS has pulled
-      # vcpython27 compiler, building a netifaces wheel is not an
-      # option anymore, so let us test on 32-bit Windows.
+      # We use netifaces, which does not ship a 64-bit wheel for the
+      # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
+      # Since MS has removed vcpython27 compiler downloads from their
+      # usual site, building a netifaces wheel is not an option
+      # anymore.  So let us just test with 32-bit Python on Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
         if: ${{ matrix.os == 'windows-latest' }}
         uses: actions/setup-python@v1

From 65398a2d63bea5e70ec65d767a7d9969d204ea82 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 26 Apr 2021 17:06:34 -0400
Subject: [PATCH 47/81] GitHub Actions: update note about Windows again

---
 .github/workflows/ci.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2adb22bc8..eeb6bbe5b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -43,8 +43,14 @@ jobs:
       # We use netifaces, which does not ship a 64-bit wheel for the
       # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
       # Since MS has removed vcpython27 compiler downloads from their
-      # usual site, building a netifaces wheel is not an option
-      # anymore.  So let us just test with 32-bit Python on Windows.
+      # usual download site, building a netifaces wheel locally is not
+      # an option anymore.  So let us just test with 32-bit Python on
+      # Windows.
+      #
+      # It seems that GitHub Actions' expression syntax is not
+      # expressive enough to select just the (32-bit Python 2.7,
+      # Windows) combination, so we'd also be switching to (32-bit
+      # Python 3.6, Windows).
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
         if: ${{ matrix.os == 'windows-latest' }}
         uses: actions/setup-python@v1

From 73d3295a6ae75de170f629821d315510208c6231 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 27 Apr 2021 15:49:25 -0400
Subject: [PATCH 48/81] GitHub Actions: use 64-bit Python 3.6 to test on
 Windows

---
 .github/workflows/ci.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index eeb6bbe5b..aa128d4ef 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -40,19 +40,25 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # See note below about need for using 32-bit Python 2.7 on
+      # Windows.  The extra handling here for Python 3.6 on Windows is
+      # because I could not figure out the right GitHub Actions
+      # expression to do this in a better way.
+      - name: Set up Python ${{ matrix.python-version }} [Windows x64]
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.6' }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
       # We use netifaces, which does not ship a 64-bit wheel for the
       # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
       # Since MS has removed vcpython27 compiler downloads from their
       # usual download site, building a netifaces wheel locally is not
       # an option anymore.  So let us just test with 32-bit Python on
       # Windows.
-      #
-      # It seems that GitHub Actions' expression syntax is not
-      # expressive enough to select just the (32-bit Python 2.7,
-      # Windows) combination, so we'd also be switching to (32-bit
-      # Python 3.6, Windows).
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' }}
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From 3722b8f628472a4c509a78267ae259438046c749 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 27 Apr 2021 15:52:28 -0400
Subject: [PATCH 49/81] GitHub Actions: quote architecture

Docs for actions/setup-python seem to do that, although
inconsistently.
---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa128d4ef..73294b730 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,7 +49,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: 'x64'
 
       # We use netifaces, which does not ship a 64-bit wheel for the
       # Python 2.7 + Windows combination, but it ships a 32-bit wheel.
@@ -62,7 +62,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x86
+          architecture: 'x86'
 
       # To use pip caching with GitHub Actions in an OS-independent
       # manner, we need `pip cache dir` command, which became
@@ -211,7 +211,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x86
+          architecture: 'x86'
 
       - name: Get pip cache directory
         id: pip-cache
@@ -275,7 +275,7 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x86
+          architecture: 'x86'
 
       - name: Get pip cache directory
         id: pip-cache

From 6d1b95b965f16f7a15a238f8431fa60cb68f9cdf Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 27 Apr 2021 16:21:22 -0400
Subject: [PATCH 50/81] GitHub Actions: add more parenthesis

https://github.community/t/and-operator-in-if-condition/154825
suggests that adding more parens might work.
---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 73294b730..f70432267 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,7 +45,7 @@ jobs:
       # because I could not figure out the right GitHub Actions
       # expression to do this in a better way.
       - name: Set up Python ${{ matrix.python-version }} [Windows x64]
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.6' }}
+        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '3.6' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
@@ -58,7 +58,7 @@ jobs:
       # an option anymore.  So let us just test with 32-bit Python on
       # Windows.
       - name: Set up Python ${{ matrix.python-version }} [Windows x86]
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '2.7' }}
+        if: ${{ ( matrix.os == 'windows-latest' ) && ( matrix.python-version == '2.7' ) }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}

From 9137da5483c9dddc62f3e7ab639151c3d4472c44 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 10:16:41 -0400
Subject: [PATCH 51/81] Stick to Unicode when possible.

---
 src/allmydata/scripts/tahoe_check.py | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index d8b7c9bce..494c1dba3 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -53,14 +53,11 @@ def check_location(options, where):
     if resp.status != 200:
         print(format_http_error("ERROR", resp), file=stderr)
         return 1
-    jdata = resp.read()
+    jdata = resp.read().decode()
+
     if options.get("raw"):
-        if PY3:
-            stdoutb = stdout.buffer
-        else:
-            stdoutb = stdout
-        stdoutb.write(jdata)
-        stdoutb.write(b"\n")
+        stdout.write(jdata)
+        stdout.write("\n")
         return 0
     data = json.loads(jdata)
 
@@ -323,17 +320,12 @@ class DeepCheckStreamer(LineOnlyReceiver, object):
             return 1
 
         # use Twisted to split this into lines
-        if PY3:
-            stdoutb = stdout.buffer
-        else:
-            stdoutb = stdout
-
         while True:
             chunk = resp.read(100)
             if not chunk:
                 break
             if self.options["raw"]:
-                stdoutb.write(chunk)
+                stdout.write(chunk.decode())
             else:
                 output.dataReceived(chunk)
         if not self.options["raw"]:

From 72a85ba62422b3e398459bde3ea6ad8cbd2fafef Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 10:19:59 -0400
Subject: [PATCH 52/81] Fix lint.

---
 src/allmydata/scripts/tahoe_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/tahoe_check.py b/src/allmydata/scripts/tahoe_check.py
index 494c1dba3..82885d073 100644
--- a/src/allmydata/scripts/tahoe_check.py
+++ b/src/allmydata/scripts/tahoe_check.py
@@ -4,7 +4,7 @@ from urllib.parse import quote as url_quote
 import json
 
 # Python 2 compatibility
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401
 

From 2f6535e26e09f0bb7e5a38d2e895d5966a0bdfd3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 10:33:59 -0400
Subject: [PATCH 53/81] First test passes on Python 3.

---
 src/allmydata/scripts/tahoe_webopen.py      |  8 +++++--
 src/allmydata/test/cli/test_create_alias.py | 26 +++++++++++----------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_webopen.py b/src/allmydata/scripts/tahoe_webopen.py
index a7b7ca7e1..0292e0d40 100644
--- a/src/allmydata/scripts/tahoe_webopen.py
+++ b/src/allmydata/scripts/tahoe_webopen.py
@@ -1,7 +1,10 @@
+from past.builtins import unicode
+
+from urllib.parse import quote as url_quote
 
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
-import urllib
+
 
 def webopen(options, opener=None):
     nodeurl = options['node-url']
@@ -15,9 +18,10 @@ def webopen(options, opener=None):
         except UnknownAliasError as e:
             e.display(stderr)
             return 1
+        path = unicode(path, "utf-8")
         if path == '/':
             path = ''
-        url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+        url = nodeurl + "uri/%s" % url_quote(rootcap)
         if path:
             url += "/" + escape_path(path)
     else:
diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py
index ea3200e2e..2718a706f 100644
--- a/src/allmydata/test/cli/test_create_alias.py
+++ b/src/allmydata/test/cli/test_create_alias.py
@@ -1,7 +1,9 @@
+from future.builtins import str
 from six.moves import StringIO
 import os.path
 from twisted.trial import unittest
-import urllib
+from urllib.parse import quote as url_quote
+
 from allmydata.util import fileutil
 from allmydata.scripts.common import get_aliases
 from allmydata.scripts import cli, runner
@@ -22,7 +24,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
         rc = cli.webopen(o.subOptions, urls.append)
         self.failUnlessReallyEqual(rc, 0)
         self.failUnlessReallyEqual(len(urls), 1)
-        self.failUnlessReallyEqual(urls[0], expected_url)
+        self.assertEqual(urls[0], expected_url)
 
     def test_create(self):
         self.basedir = "cli/CreateAlias/create"
@@ -36,19 +38,19 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.assertIn("Alias 'tahoe' created", stdout)
             aliases = get_aliases(self.get_clientdir())
             self.failUnless("tahoe" in aliases)
-            self.failUnless(aliases["tahoe"].startswith("URI:DIR2:"))
+            self.failUnless(aliases["tahoe"].startswith(b"URI:DIR2:"))
         d.addCallback(_done)
         d.addCallback(lambda res: self.do_cli("create-alias", "two:"))
 
         def _stash_urls(res):
             aliases = get_aliases(self.get_clientdir())
             node_url_file = os.path.join(self.get_clientdir(), "node.url")
-            nodeurl = fileutil.read(node_url_file).strip()
+            nodeurl = fileutil.read(node_url_file, mode="r").strip()
             self.welcome_url = nodeurl
             uribase = nodeurl + "uri/"
-            self.tahoe_url = uribase + urllib.quote(aliases["tahoe"])
+            self.tahoe_url = uribase + url_quote(aliases["tahoe"])
             self.tahoe_subdir_url = self.tahoe_url + "/subdir"
-            self.two_url = uribase + urllib.quote(aliases["two"])
+            self.two_url = uribase + url_quote(aliases["two"])
             self.two_uri = aliases["two"]
         d.addCallback(_stash_urls)
 
@@ -128,13 +130,13 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             # like a valid dircap, so get_aliases() will raise an exception.
             aliases = get_aliases(self.get_clientdir())
             self.failUnless("added" in aliases)
-            self.failUnless(aliases["added"].startswith("URI:DIR2:"))
+            self.failUnless(aliases["added"].startswith(b"URI:DIR2:"))
             # to be safe, let's confirm that we don't see "NAME2:" in CAP1.
             # No chance of a false-negative, because the hyphen in
             # "un-corrupted1" is not a valid base32 character.
-            self.failIfIn("un-corrupted1:", aliases["added"])
+            self.failIfIn(b"un-corrupted1:", aliases["added"])
             self.failUnless("un-corrupted1" in aliases)
-            self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:"))
+            self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:"))
         d.addCallback(_check_not_corrupted1)
 
         def _remove_trailing_newline_and_add_alias(ign):
@@ -149,10 +151,10 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failIf(stderr)
             aliases = get_aliases(self.get_clientdir())
             self.failUnless("un-corrupted1" in aliases)
-            self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:"))
-            self.failIfIn("un-corrupted2:", aliases["un-corrupted1"])
+            self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:"))
+            self.failIfIn(b"un-corrupted2:", aliases["un-corrupted1"])
             self.failUnless("un-corrupted2" in aliases)
-            self.failUnless(aliases["un-corrupted2"].startswith("URI:DIR2:"))
+            self.failUnless(aliases["un-corrupted2"].startswith(b"URI:DIR2:"))
         d.addCallback(_check_not_corrupted)
         return d
 

From 46c03f6b75ffe6799af751ba8eedf92c7d35cf99 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:05:56 -0400
Subject: [PATCH 54/81] Remove duplication, and add support for testing cases
 where stdin/stdout/stderr have to be bytes.

---
 src/allmydata/test/common_util.py | 76 ++++++++++++++++++-------------
 1 file changed, 44 insertions(+), 32 deletions(-)

diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 16f945239..95065710c 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -6,7 +6,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import unicode_literals
 
-from future.utils import PY2, bchr, binary_type
+from future.utils import PY2, PY3, bchr, binary_type
 from future.builtins import str as future_str
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min  # noqa: F401
@@ -15,7 +15,8 @@ import os
 import time
 import signal
 from random import randrange
-from six.moves import StringIO
+if PY2:
+    from StringIO import StringIO
 from io import (
     TextIOWrapper,
     BytesIO,
@@ -64,23 +65,28 @@ def run_cli_native(verb, *args, **kwargs):
     Most code should prefer ``run_cli_unicode`` which deals with all the
     necessary encoding considerations.
 
-    :param native_str verb: The command to run.  For example, ``"create-node"``.
+    :param native_str verb: The command to run.  For example,
+        ``"create-node"``.
 
-    :param [native_str] args: The arguments to pass to the command.  For example,
-        ``("--hostname=localhost",)``.
+    :param [native_str] args: The arguments to pass to the command.  For
+        example, ``("--hostname=localhost",)``.
 
-    :param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable
-        before ``verb``.
+    :param [native_str] nodeargs: Extra arguments to pass to the Tahoe
+        executable before ``verb``.
 
-    :param native_str stdin: Text to pass to the command via stdin.
+    :param bytes|unicode stdin: Text or bytes to pass to the command via stdin.
 
     :param NoneType|str encoding: The name of an encoding which stdout and
-        stderr will be configured to use.  ``None`` means stdout and stderr
-        will accept bytes and unicode and use the default system encoding for
-        translating between them.
+        stderr will be configured to use.  ``None`` means matching default
+        behavior for the given Python version.
+
+    :param bool return_bytes: If False, stdout/stderr is native string,
+        matching native behavior.  If True, stdout/stderr are returned as
+        bytes.
     """
     nodeargs = kwargs.pop("nodeargs", [])
-    encoding = kwargs.pop("encoding", None)
+    encoding = kwargs.pop("encoding", "utf-8")
+    return_bytes = kwargs.pop("return_bytes", False)
     verb = maybe_unicode_to_argv(verb)
     args = [maybe_unicode_to_argv(a) for a in args]
     nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs]
@@ -93,36 +99,42 @@ def run_cli_native(verb, *args, **kwargs):
     )
     argv = nodeargs + [verb] + list(args)
     stdin = kwargs.get("stdin", "")
-    if encoding is None:
-        if PY2:
-            # The original behavior, the Python 2 behavior, is to accept either
-            # bytes or unicode and try to automatically encode or decode as
-            # necessary.  This works okay for ASCII and if LANG is set
-            # appropriately.  These aren't great constraints so we should move
-            # away from this behavior.
-            stdout = StringIO()
-            stderr = StringIO()
-        else:
-            # Default on Python 3 is accepting text.
-            stdout = TextIOWrapper(BytesIO(), "utf-8")
-            stderr = TextIOWrapper(BytesIO(), "utf-8")
+    if PY2:
+        # The original behavior, the Python 2 behavior, is to accept either
+        # bytes or unicode and try to automatically encode or decode as
+        # necessary.  This works okay for ASCII and if LANG is set
+        # appropriately.  These aren't great constraints so we should move
+        # away from this behavior.
+        stdin = StringIO(stdin)
+        stdout = StringIO()
+        stderr = StringIO()
     else:
         # The new behavior, the Python 3 behavior, is to accept unicode and
-        # encode it using a specific encoding.  For older versions of Python
-        # 3, the encoding is determined from LANG (bad) but for newer Python
-        # 3, the encoding is always utf-8 (good).  Tests can pass in different
-        # encodings to exercise different behaviors.
+        # encode it using a specific encoding. For older versions of Python 3,
+        # the encoding is determined from LANG (bad) but for newer Python 3,
+        # the encoding is either LANG if it supports full Unicode, otherwise
+        # utf-8 (good). Tests can pass in different encodings to exercise
+        # different behaviors.
+        if isinstance(stdin, str):
+            stdin = stdin.encode(encoding)
+        stdin = TextIOWrapper(BytesIO(stdin), encoding)
         stdout = TextIOWrapper(BytesIO(), encoding)
         stderr = TextIOWrapper(BytesIO(), encoding)
     d = defer.succeed(argv)
     d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout)
     d.addCallback(runner.dispatch,
-                  stdin=StringIO(stdin),
+                  stdin=stdin,
                   stdout=stdout, stderr=stderr)
-    def _done(rc):
+    def _done(rc, stdout=stdout, stderr=stderr):
+        if return_bytes and PY3:
+            stdout = stdout.buffer
+            stderr = stderr.buffer
         return 0, _getvalue(stdout), _getvalue(stderr)
-    def _err(f):
+    def _err(f, stdout=stdout, stderr=stderr):
         f.trap(SystemExit)
+        if return_bytes and PY3:
+            stdout = stdout.buffer
+            stderr = stderr.buffer
         return f.value.code, _getvalue(stdout), _getvalue(stderr)
     d.addCallbacks(_done, _err)
     return d

From 3846df8e4fb3a8677d77ebc7ae62b9d8b1079507 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:06:31 -0400
Subject: [PATCH 55/81] All test_create_alias tests pass on Python 3.

---
 src/allmydata/scripts/tahoe_put.py          | 12 ++++++--
 src/allmydata/test/cli/test_create_alias.py | 31 ++++++++++-----------
 2 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_put.py b/src/allmydata/scripts/tahoe_put.py
index b64283a81..8db705d01 100644
--- a/src/allmydata/scripts/tahoe_put.py
+++ b/src/allmydata/scripts/tahoe_put.py
@@ -1,8 +1,9 @@
 from __future__ import print_function
 
+from future.utils import PY2
 from past.builtins import unicode
 
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from urllib.parse import quote as url_quote
 
 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
@@ -83,8 +84,13 @@ def put(options):
         # Content-Length field. So we currently must copy it.
         if verbosity > 0:
             print("waiting for file data on stdin..", file=stderr)
-        data = stdin.read()
-        infileobj = StringIO(data)
+        # We're uploading arbitrary files, so this had better be bytes:
+        if PY2:
+            stdinb = stdin
+        else:
+            stdinb = stdin.buffer
+        data = stdinb.read()
+        infileobj = BytesIO(data)
 
     resp = do_http("PUT", url, infileobj)
 
diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py
index 2718a706f..30f613486 100644
--- a/src/allmydata/test/cli/test_create_alias.py
+++ b/src/allmydata/test/cli/test_create_alias.py
@@ -162,48 +162,47 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/CreateAlias/create_unicode"
         self.set_up_grid(oneshare=True)
 
-        try:
-            etudes_arg = u"\u00E9tudes".encode(get_io_encoding())
-            lumiere_arg = u"lumi\u00E8re.txt".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
+        etudes_arg = u"\u00E9tudes"
+        lumiere_arg = u"lumi\u00E8re.txt"
 
         d = self.do_cli("create-alias", etudes_arg)
         def _check_create_unicode(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out)
 
             aliases = get_aliases(self.get_clientdir())
-            self.failUnless(aliases[u"\u00E9tudes"].startswith("URI:DIR2:"))
+            self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:"))
         d.addCallback(_check_create_unicode)
 
         d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":"))
         def _check_ls1(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(err), 0, err)
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check_ls1)
 
+        DATA = b"Blah blah blah \xff blah \x00 blah"
         d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":uploaded.txt",
-                                              stdin="Blah blah blah"))
+                                              stdin=DATA))
 
         d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":"))
         def _check_ls2(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(out, "uploaded.txt\n")
         d.addCallback(_check_ls2)
 
-        d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt"))
+        d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt",
+                                              return_bytes=True))
         def _check_get(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
-            self.failUnlessReallyEqual(out, "Blah blah blah")
+            self.assertEqual(len(err), 0, err)
+            self.failUnlessReallyEqual(out, DATA)
         d.addCallback(_check_get)
 
         # Ensure that an Unicode filename in an Unicode alias works as expected
@@ -211,11 +210,11 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
                                               stdin="Let the sunshine In!"))
 
         d.addCallback(lambda res: self.do_cli("get",
-                                              get_aliases(self.get_clientdir())[u"\u00E9tudes"] + "/" + lumiere_arg))
+                                              str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg))
         def _check_get2(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(out, "Let the sunshine In!")
         d.addCallback(_check_get2)
 

From da8e0d61aa7b7a695ab69a72f43071b76cc81548 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:15:48 -0400
Subject: [PATCH 56/81] Port to Python 3.

---
 src/allmydata/test/cli/test_create_alias.py | 28 +++++++++++++++------
 src/allmydata/util/_python3.py              |  1 +
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py
index 30f613486..e82ecf60c 100644
--- a/src/allmydata/test/cli/test_create_alias.py
+++ b/src/allmydata/test/cli/test_create_alias.py
@@ -1,4 +1,16 @@
-from future.builtins import str
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from six import ensure_str
 from six.moves import StringIO
 import os.path
 from twisted.trial import unittest
@@ -170,7 +182,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out)
+            self.failUnlessIn(ensure_str("Alias %s created") % quote_output(etudes_arg), out)
 
             aliases = get_aliases(self.get_clientdir())
             self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:"))
@@ -193,7 +205,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessReallyEqual(out, "uploaded.txt\n")
+            self.assertEqual(out, "uploaded.txt\n")
         d.addCallback(_check_ls2)
 
         d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt",
@@ -207,15 +219,17 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         # Ensure that an Unicode filename in an Unicode alias works as expected
         d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":" + lumiere_arg,
-                                              stdin="Let the sunshine In!"))
+                                              stdin=b"Let the sunshine In!"))
 
-        d.addCallback(lambda res: self.do_cli("get",
-                                              str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg))
+        d.addCallback(lambda res: self.do_cli(
+            "get",
+            str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg,
+            return_bytes=True))
         def _check_get2(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessReallyEqual(out, "Let the sunshine In!")
+            self.failUnlessReallyEqual(out, b"Let the sunshine In!")
         d.addCallback(_check_get2)
 
         return d
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 471c7e913..1e6111e6c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -179,6 +179,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_check",
     "allmydata.test.cli.test_cp",
     "allmydata.test.cli.test_create",
+    "allmydata.test.cli.test_create_alias",
     "allmydata.test.cli.test_invite",
     "allmydata.test.cli.test_status",
 

From 4eb9be1996b8fa091e26b16a6882e99ae74f6688 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:17:09 -0400
Subject: [PATCH 57/81] News file.

---
 newsfragments/3687.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3687.minor

diff --git a/newsfragments/3687.minor b/newsfragments/3687.minor
new file mode 100644
index 000000000..e69de29bb

From 463f9fe80266d2a8934a92e549503bf06ae9c68c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:31:25 -0400
Subject: [PATCH 58/81] Tests pass on Python 3.

---
 src/allmydata/test/cli/test_put.py | 57 +++++++++++++++++-------------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py
index 3392e67b4..ef26f5990 100644
--- a/src/allmydata/test/cli/test_put.py
+++ b/src/allmydata/test/cli/test_put.py
@@ -1,3 +1,5 @@
+from past.builtins import unicode
+
 import os.path
 from twisted.trial import unittest
 from twisted.python import usage
@@ -17,7 +19,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         # tahoe get `echo DATA | tahoe put`
         # tahoe get `echo DATA | tahoe put -`
         self.basedir = "cli/Put/unlinked_immutable_stdin"
-        DATA = "data" * 100
+        DATA = b"data\xff" * 100
         self.set_up_grid(oneshare=True)
         d = self.do_cli("put", stdin=DATA)
         def _uploaded(res):
@@ -27,10 +29,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.readcap = out
             self.failUnless(self.readcap.startswith("URI:CHK:"))
         d.addCallback(_uploaded)
-        d.addCallback(lambda res: self.do_cli("get", self.readcap))
+        d.addCallback(lambda res: self.do_cli("get", self.readcap,
+                                              return_bytes=True))
         def _downloaded(res):
             (rc, out, err) = res
-            self.failUnlessReallyEqual(err, "")
+            self.failUnlessReallyEqual(err, b"")
             self.failUnlessReallyEqual(out, DATA)
         d.addCallback(_downloaded)
         d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA))
@@ -49,7 +52,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         rel_fn = unicode(os.path.join(self.basedir, "DATAFILE"))
         abs_fn = abspath_expanduser_unicode(rel_fn)
         # we make the file small enough to fit in a LIT file, for speed
-        fileutil.write(rel_fn, "short file")
+        fileutil.write(rel_fn, b"short file has some bytes \xff yes")
         d = self.do_cli_unicode(u"put", [rel_fn])
         def _uploaded(args):
             (rc, out, err) = args
@@ -79,8 +82,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         rel_fn = os.path.join(self.basedir, "DATAFILE")
         # we make the file small enough to fit in a LIT file, for speed
-        DATA = "short file"
-        DATA2 = "short file two"
+        DATA = b"short file"
+        DATA2 = b"short file two"
         fileutil.write(rel_fn, DATA)
 
         d = self.do_cli("create-alias", "tahoe")
@@ -95,7 +98,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.readcap = readcap
         d.addCallback(_uploaded)
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:uploaded.txt"))
+                      self.do_cli("get", "tahoe:uploaded.txt",
+                                  return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
@@ -110,32 +114,36 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         d.addCallback(lambda res:
                       self.do_cli("put", rel_fn, "subdir/uploaded2.txt"))
-        d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt"))
+        d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt",
+                                              return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
         d.addCallback(lambda res:
                       self.do_cli("put", rel_fn, "tahoe:uploaded3.txt"))
-        d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt"))
+        d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt",
+                                              return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
         d.addCallback(lambda res:
                       self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt"))
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:subdir/uploaded4.txt"))
+                      self.do_cli("get", "tahoe:subdir/uploaded4.txt",
+                                  return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
         def _get_dircap(res):
-            self.dircap = get_aliases(self.get_clientdir())["tahoe"]
+            self.dircap = unicode(get_aliases(self.get_clientdir())["tahoe"], "ascii")
         d.addCallback(_get_dircap)
 
         d.addCallback(lambda res:
                       self.do_cli("put", rel_fn,
                                   self.dircap+":./uploaded5.txt"))
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:uploaded5.txt"))
+                      self.do_cli("get", "tahoe:uploaded5.txt",
+                                  return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
@@ -143,7 +151,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.do_cli("put", rel_fn,
                                   self.dircap+":./subdir/uploaded6.txt"))
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:subdir/uploaded6.txt"))
+                      self.do_cli("get", "tahoe:subdir/uploaded6.txt",
+                                  return_bytes=True))
         d.addCallback(lambda rc_stdout_stderr:
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
@@ -158,10 +167,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Put/mutable_unlinked"
         self.set_up_grid(oneshare=True)
 
-        DATA = "data" * 100
-        DATA2 = "two" * 100
+        DATA = b"data" * 100
+        DATA2 = b"two" * 100
         rel_fn = os.path.join(self.basedir, "DATAFILE")
-        DATA3 = "three" * 100
+        DATA3 = b"three" * 100
         fileutil.write(rel_fn, DATA3)
 
         d = self.do_cli("put", "--mutable", stdin=DATA)
@@ -172,7 +181,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.filecap = out
             self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap)
         d.addCallback(_created)
-        d.addCallback(lambda res: self.do_cli("get", self.filecap))
+        d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
         d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA))
 
         d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2))
@@ -182,7 +191,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessIn("200 OK", err)
             self.failUnlessReallyEqual(self.filecap, out)
         d.addCallback(_replaced)
-        d.addCallback(lambda res: self.do_cli("get", self.filecap))
+        d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
         d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
 
         d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap))
@@ -191,7 +200,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessIn("200 OK", err)
             self.failUnlessReallyEqual(self.filecap, out)
         d.addCallback(_replaced2)
-        d.addCallback(lambda res: self.do_cli("get", self.filecap))
+        d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
         d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA3))
 
         return d
@@ -436,10 +445,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
     def test_immutable_from_file_unicode(self):
         # tahoe put "\u00E0 trier.txt" "\u00E0 trier.txt"
 
-        try:
-            a_trier_arg = u"\u00E0 trier.txt".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
+        a_trier_arg = u"\u00E0 trier.txt"
 
         skip_if_cannot_represent_filename(u"\u00E0 trier.txt")
 
@@ -448,7 +454,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt")
         # we make the file small enough to fit in a LIT file, for speed
-        DATA = "short file"
+        DATA = b"short file \xff bytes"
         fileutil.write(rel_fn, DATA)
 
         d = self.do_cli("create-alias", "tahoe")
@@ -464,7 +470,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_uploaded)
 
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:" + a_trier_arg))
+                      self.do_cli("get", "tahoe:" + a_trier_arg,
+                                  return_bytes=True))
         d.addCallback(lambda rc_out_err:
                       self.failUnlessReallyEqual(rc_out_err[1], DATA))
 

From f6b5628ce120c72066bd3870fae0a6c57174cdac Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 30 Apr 2021 11:33:51 -0400
Subject: [PATCH 59/81] Port to Python 3.

---
 src/allmydata/test/cli/test_put.py | 26 ++++++++++++++++++--------
 src/allmydata/util/_python3.py     |  1 +
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/cli/test_put.py b/src/allmydata/test/cli/test_put.py
index ef26f5990..c6a577074 100644
--- a/src/allmydata/test/cli/test_put.py
+++ b/src/allmydata/test/cli/test_put.py
@@ -1,4 +1,14 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os.path
 from twisted.trial import unittest
@@ -49,7 +59,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Put/unlinked_immutable_from_file"
         self.set_up_grid(oneshare=True)
 
-        rel_fn = unicode(os.path.join(self.basedir, "DATAFILE"))
+        rel_fn = str(os.path.join(self.basedir, "DATAFILE"))
         abs_fn = abspath_expanduser_unicode(rel_fn)
         # we make the file small enough to fit in a LIT file, for speed
         fileutil.write(rel_fn, b"short file has some bytes \xff yes")
@@ -135,7 +145,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
                       self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
 
         def _get_dircap(res):
-            self.dircap = unicode(get_aliases(self.get_clientdir())["tahoe"], "ascii")
+            self.dircap = str(get_aliases(self.get_clientdir())["tahoe"], "ascii")
         d.addCallback(_get_dircap)
 
         d.addCallback(lambda res:
@@ -213,10 +223,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Put/mutable"
         self.set_up_grid(oneshare=True)
 
-        DATA1 = "data" * 100
+        DATA1 = b"data" * 100
         fn1 = os.path.join(self.basedir, "DATA1")
         fileutil.write(fn1, DATA1)
-        DATA2 = "two" * 100
+        DATA2 = b"two\xff" * 100
         fn2 = os.path.join(self.basedir, "DATA2")
         fileutil.write(fn2, DATA2)
 
@@ -238,7 +248,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessEqual(out, self.uri, str(res))
         d.addCallback(_check2)
         d.addCallback(lambda res:
-                      self.do_cli("get", "tahoe:uploaded.txt"))
+                      self.do_cli("get", "tahoe:uploaded.txt", return_bytes=True))
         d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
         return d
 
@@ -438,7 +448,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         return d
 
@@ -452,7 +462,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Put/immutable_from_file_unicode"
         self.set_up_grid(oneshare=True)
 
-        rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt")
+        rel_fn = os.path.join(str(self.basedir), u"\u00E0 trier.txt")
         # we make the file small enough to fit in a LIT file, for speed
         DATA = b"short file \xff bytes"
         fileutil.write(rel_fn, DATA)
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 1e6111e6c..6695b685f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -181,6 +181,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_create",
     "allmydata.test.cli.test_create_alias",
     "allmydata.test.cli.test_invite",
+    "allmydata.test.cli.test_put",
     "allmydata.test.cli.test_status",
 
     "allmydata.test.mutable.test_checker",

From b3ede6b9f232338b0d5ed6a3f4650d7648df2ab9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 10:25:48 -0400
Subject: [PATCH 60/81] Nicer way to say the same thing.

---
 src/allmydata/scripts/cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py
index 826c36e1f..645b57f41 100644
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions):
     def parseArgs(self, *args):
         if len(args) < 2:
             raise usage.UsageError("cp requires at least two arguments")
-        self.sources = list(map(argv_to_unicode, args[:-1]))
+        self.sources = [argv_to_unicode(arg) for arg in args[:-1]]
         self.destination = argv_to_unicode(args[-1])
 
     synopsis = "[options] FROM.. TO"

From 99543877d6d0e45c55230249548749fb1e3e5666 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 10:27:43 -0400
Subject: [PATCH 61/81] Fix flake.

---
 src/allmydata/test/cli/test_create_alias.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_create_alias.py b/src/allmydata/test/cli/test_create_alias.py
index e82ecf60c..4a252f372 100644
--- a/src/allmydata/test/cli/test_create_alias.py
+++ b/src/allmydata/test/cli/test_create_alias.py
@@ -20,7 +20,7 @@ from allmydata.util import fileutil
 from allmydata.scripts.common import get_aliases
 from allmydata.scripts import cli, runner
 from ..no_network import GridTestMixin
-from allmydata.util.encodingutil import quote_output, get_io_encoding
+from allmydata.util.encodingutil import quote_output
 from .common import CLITestMixin
 
 class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):

From c275f9ae54a5d256655824ec1099b4610a63cd60 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 10:47:30 -0400
Subject: [PATCH 62/81] Tests pass on Python 3.

---
 src/allmydata/scripts/tahoe_ls.py   |  3 +-
 src/allmydata/test/cli/test_list.py | 62 +++++++++++++----------------
 2 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py
index 71db3f5cb..92d5adfef 100644
--- a/src/allmydata/scripts/tahoe_ls.py
+++ b/src/allmydata/scripts/tahoe_ls.py
@@ -64,13 +64,14 @@ def list(options):
         print(quote_output(data, quotemarks=False), file=stderr)
         return 1
 
+    path = unicode(path, "utf-8")
     nodetype, d = parsed
     children = {}
     if nodetype == "dirnode":
         children = d['children']
     else:
         # paths returned from get_alias are always valid UTF-8
-        childname = path.split("/")[-1].decode('utf-8')
+        childname = path.split("/")[-1]
         children = {childname: (nodetype, d)}
         if "metadata" not in d:
             d["metadata"] = {}
diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py
index fff57cdc9..68571b49b 100644
--- a/src/allmydata/test/cli/test_list.py
+++ b/src/allmydata/test/cli/test_list.py
@@ -1,3 +1,6 @@
+from future.utils import PY3
+from past.builtins import unicode
+
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -8,30 +11,26 @@ from ..no_network import GridTestMixin
 from allmydata.util.encodingutil import quote_output, get_io_encoding
 from .common import CLITestMixin
 
+
 class List(GridTestMixin, CLITestMixin, unittest.TestCase):
     def test_list(self):
         self.basedir = "cli/List/list"
         self.set_up_grid()
         c0 = self.g.clients[0]
-        small = "small"
+        small = b"small"
 
-        # u"g\u00F6\u00F6d" might not be representable in the argv and/or output encodings.
-        # It is initially included in the directory in any case.
-        try:
-            good_arg = u"g\u00F6\u00F6d".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            good_arg = None
+        good_arg = u"g\u00F6\u00F6d"
+        good_out = u"g\u00F6\u00F6d"
 
-        try:
-            good_out = u"g\u00F6\u00F6d".encode(get_io_encoding())
-        except UnicodeEncodeError:
-            good_out = None
+        # On Python 2 we get bytes, so we need encoded version. On Python 3
+        # stdio is unicode so can leave unchanged.
+        good_out_encoded = good_out if PY3 else good_out.encode(get_io_encoding())
 
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
             self.rootnode = n
-            self.rooturi = n.get_uri()
-            return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=""))
+            self.rooturi = unicode(n.get_uri(), "utf-8")
+            return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_goodcap(n):
             self.goodcap = n.get_uri()
@@ -47,15 +46,10 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("ls"))
         def _check1(args):
             (rc, out, err) = args
-            if good_out is None:
-                self.failUnlessReallyEqual(rc, 1)
-                self.failUnlessIn("files whose names could not be converted", err)
-                self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err)
-                self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share"]))
-            else:
-                self.failUnlessReallyEqual(rc, 0)
-                self.failUnlessReallyEqual(err, "")
-                self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", good_out]))
+            self.failUnlessReallyEqual(rc, 0)
+            self.assertEqual(len(err), 0, err)
+            self.failUnlessReallyEqual(sorted(out.splitlines()),
+                                       sorted(["0share", "1share", good_out_encoded]))
         d.addCallback(_check1)
         d.addCallback(lambda ign: self.do_cli("ls", "missing"))
         def _check2(args):
@@ -87,7 +81,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
                 # listing a file (as dir/filename) should have the edge metadata,
                 # including the filename
                 self.failUnlessReallyEqual(rc, 0)
-                self.failUnlessIn(good_out, out)
+                self.failUnlessIn(good_out_encoded, out)
                 self.failIfIn("-r-- %d -" % len(small), out,
                               "trailing hyphen means unknown date")
 
@@ -139,7 +133,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("ls", "-l", self.rooturi + ":./good"))
         d.addCallback(_check4_ascii)
 
-        unknown_immcap = "imm.URI:unknown"
+        unknown_immcap = b"imm.URI:unknown"
         def _create_unknown(ign):
             nm = c0.nodemaker
             kids = {u"unknownchild-imm": (nm.create_from_cap(unknown_immcap), {})}
@@ -226,8 +220,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             # The uploaders may run at the same time, so we need two
             # MutableData instances or they'll fight over offsets &c and
             # break.
-            mutable_data = MutableData("data" * 100000)
-            mutable_data2 = MutableData("data" * 100000)
+            mutable_data = MutableData(b"data" * 100000)
+            mutable_data2 = MutableData(b"data" * 100000)
             # Add both kinds of mutable node.
             d1 = nm.create_mutable_file(mutable_data,
                                         version=MDMF_VERSION)
@@ -235,8 +229,8 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
                                         version=SDMF_VERSION)
             # Add an immutable node. We do this through the directory,
             # with add_file.
-            immutable_data = upload.Data("immutable data" * 100000,
-                                         convergence="")
+            immutable_data = upload.Data(b"immutable data" * 100000,
+                                         convergence=b"")
             d3 = n.add_file(u"immutable", immutable_data)
             ds = [d1, d2, d3]
             dl = defer.DeferredList(ds)
@@ -294,12 +288,12 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _got_json(args):
             (rc, out, err) = args
             self.failUnlessEqual(rc, 0)
-            self.failUnlessEqual(err, "")
-            self.failUnlessIn(self._mdmf_uri, out)
-            self.failUnlessIn(self._mdmf_readonly_uri, out)
-            self.failUnlessIn(self._sdmf_uri, out)
-            self.failUnlessIn(self._sdmf_readonly_uri, out)
-            self.failUnlessIn(self._imm_uri, out)
+            self.assertEqual(len(err), 0, err)
+            self.failUnlessIn(unicode(self._mdmf_uri, "ascii"), out)
+            self.failUnlessIn(unicode(self._mdmf_readonly_uri, "ascii"), out)
+            self.failUnlessIn(unicode(self._sdmf_uri, "ascii"), out)
+            self.failUnlessIn(unicode(self._sdmf_readonly_uri, "ascii"), out)
+            self.failUnlessIn(unicode(self._imm_uri, "ascii"), out)
             self.failUnlessIn('"format": "SDMF"', out)
             self.failUnlessIn('"format": "MDMF"', out)
         d.addCallback(_got_json)

From 2b751c44dbf3091ba1b213e0298d8a0b75fbbf15 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:10:54 -0400
Subject: [PATCH 63/81] Port to Python 3.

---
 src/allmydata/test/cli/test_list.py | 52 +++++++++++++++++------------
 src/allmydata/util/_python3.py      |  1 +
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/src/allmydata/test/cli/test_list.py b/src/allmydata/test/cli/test_list.py
index 68571b49b..1206579f1 100644
--- a/src/allmydata/test/cli/test_list.py
+++ b/src/allmydata/test/cli/test_list.py
@@ -1,5 +1,15 @@
-from future.utils import PY3
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, PY3
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_str
 
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -29,7 +39,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         d = c0.create_dirnode()
         def _stash_root_and_create_file(n):
             self.rootnode = n
-            self.rooturi = unicode(n.get_uri(), "utf-8")
+            self.rooturi = str(n.get_uri(), "utf-8")
             return n.add_file(u"g\u00F6\u00F6d", upload.Data(small, convergence=b""))
         d.addCallback(_stash_root_and_create_file)
         def _stash_goodcap(n):
@@ -37,10 +47,10 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_stash_goodcap)
         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"1share"))
         d.addCallback(lambda n:
-                      self.delete_shares_numbered(n.get_uri(), range(1,10)))
+                      self.delete_shares_numbered(n.get_uri(), list(range(1,10))))
         d.addCallback(lambda ign: self.rootnode.create_subdirectory(u"0share"))
         d.addCallback(lambda n:
-                      self.delete_shares_numbered(n.get_uri(), range(0,10)))
+                      self.delete_shares_numbered(n.get_uri(), list(range(0,10))))
         d.addCallback(lambda ign:
                       self.do_cli("add-alias", "tahoe", self.rooturi))
         d.addCallback(lambda ign: self.do_cli("ls"))
@@ -48,15 +58,15 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessReallyEqual(sorted(out.splitlines()),
-                                       sorted(["0share", "1share", good_out_encoded]))
+            expected = sorted([ensure_str("0share"), ensure_str("1share"), good_out_encoded])
+            self.assertEqual(sorted(out.splitlines()), expected)
         d.addCallback(_check1)
         d.addCallback(lambda ign: self.do_cli("ls", "missing"))
         def _check2(args):
             (rc, out, err) = args
             self.failIfEqual(rc, 0)
-            self.failUnlessReallyEqual(err.strip(), "No such file or directory")
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(err.strip(), "No such file or directory")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check2)
         d.addCallback(lambda ign: self.do_cli("ls", "1share"))
         def _check3(args):
@@ -66,7 +76,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessIn("UnrecoverableFileError:", err)
             self.failUnlessIn("could not be retrieved, because there were "
                               "insufficient good shares.", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check3)
         d.addCallback(lambda ign: self.do_cli("ls", "0share"))
         d.addCallback(_check3)
@@ -76,13 +86,13 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
                 self.failUnlessReallyEqual(rc, 1)
                 self.failUnlessIn("files whose names could not be converted", err)
                 self.failUnlessIn(quote_output(u"g\u00F6\u00F6d"), err)
-                self.failUnlessReallyEqual(out, "")
+                self.assertEqual(len(out), 0, out)
             else:
                 # listing a file (as dir/filename) should have the edge metadata,
                 # including the filename
                 self.failUnlessReallyEqual(rc, 0)
                 self.failUnlessIn(good_out_encoded, out)
-                self.failIfIn("-r-- %d -" % len(small), out,
+                self.failIfIn(ensure_str("-r-- %d -" % len(small)), out,
                               "trailing hyphen means unknown date")
 
         if good_arg is not None:
@@ -100,7 +110,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             # metadata, just the size
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual("-r-- %d -" % len(small), out.strip())
+            self.assertEqual("-r-- %d -" % len(small), out.strip())
         d.addCallback(lambda ign: self.do_cli("ls", "-l", self.goodcap))
         d.addCallback(_check5)
 
@@ -112,7 +122,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check1_ascii(args):
             (rc,out,err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(len(err), 0, err)
             self.failUnlessReallyEqual(sorted(out.splitlines()), sorted(["0share", "1share", "good"]))
         d.addCallback(_check1_ascii)
         def _check4_ascii(args):
@@ -172,7 +182,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         return d
 
@@ -187,7 +197,7 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("nonexistent", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         return d
 
@@ -289,11 +299,11 @@ class List(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessEqual(rc, 0)
             self.assertEqual(len(err), 0, err)
-            self.failUnlessIn(unicode(self._mdmf_uri, "ascii"), out)
-            self.failUnlessIn(unicode(self._mdmf_readonly_uri, "ascii"), out)
-            self.failUnlessIn(unicode(self._sdmf_uri, "ascii"), out)
-            self.failUnlessIn(unicode(self._sdmf_readonly_uri, "ascii"), out)
-            self.failUnlessIn(unicode(self._imm_uri, "ascii"), out)
+            self.failUnlessIn(str(self._mdmf_uri, "ascii"), out)
+            self.failUnlessIn(str(self._mdmf_readonly_uri, "ascii"), out)
+            self.failUnlessIn(str(self._sdmf_uri, "ascii"), out)
+            self.failUnlessIn(str(self._sdmf_readonly_uri, "ascii"), out)
+            self.failUnlessIn(str(self._imm_uri, "ascii"), out)
             self.failUnlessIn('"format": "SDMF"', out)
             self.failUnlessIn('"format": "MDMF"', out)
         d.addCallback(_got_json)
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 6695b685f..400a0ea9e 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -181,6 +181,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_create",
     "allmydata.test.cli.test_create_alias",
     "allmydata.test.cli.test_invite",
+    "allmydata.test.cli.test_list",
     "allmydata.test.cli.test_put",
     "allmydata.test.cli.test_status",
 

From 9dcfa2171ef11e8e7ed111a6613840d52d965fb3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:13:27 -0400
Subject: [PATCH 64/81] Tests pass on Python 3.

---
 src/allmydata/scripts/tahoe_mv.py | 3 +++
 src/allmydata/test/cli/test_mv.py | 8 ++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py
index bdaf134ff..580475c91 100644
--- a/src/allmydata/scripts/tahoe_mv.py
+++ b/src/allmydata/scripts/tahoe_mv.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 import re
 from urllib.parse import quote as url_quote
 import json
@@ -47,6 +49,7 @@ def mv(options, mode="move"):
     if path:
         to_url += "/" + escape_path(path)
 
+    from_path = unicode(from_path, "utf-8")
     if to_url.endswith("/"):
         # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt"
         to_url += escape_path(from_path[from_path.rfind("/")+1:])
diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py
index 9d1a64974..21378c353 100644
--- a/src/allmydata/test/cli/test_mv.py
+++ b/src/allmydata/test/cli/test_mv.py
@@ -10,10 +10,10 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Mv/mv_behavior"
         self.set_up_grid(oneshare=True)
         fn1 = os.path.join(self.basedir, "file1")
-        DATA1 = "Nuclear launch codes"
+        DATA1 = b"Nuclear launch codes"
         fileutil.write(fn1, DATA1)
         fn2 = os.path.join(self.basedir, "file2")
-        DATA2 = "UML diagrams"
+        DATA2 = b"UML diagrams"
         fileutil.write(fn2, DATA2)
         # copy both files to the grid
         d = self.do_cli("create-alias", "tahoe")
@@ -104,11 +104,11 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.basedir = "cli/Mv/mv_error_if_DELETE_fails"
         self.set_up_grid(oneshare=True)
         fn1 = os.path.join(self.basedir, "file1")
-        DATA1 = "Nuclear launch codes"
+        DATA1 = b"Nuclear launch codes"
         fileutil.write(fn1, DATA1)
 
         original_do_http = tahoe_mv.do_http
-        def mock_do_http(method, url, body=""):
+        def mock_do_http(method, url, body=b""):
             if method == "DELETE":
                 class FakeResponse(object):
                     def read(self):

From a7d4fed1bada5abcc96e0dfb8b15746585f2d41b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:15:10 -0400
Subject: [PATCH 65/81] Port to Python 3.

---
 src/allmydata/test/cli/test_mv.py | 17 +++++++++++++++--
 src/allmydata/util/_python3.py    |  1 +
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/cli/test_mv.py b/src/allmydata/test/cli/test_mv.py
index 21378c353..0bb9ba369 100644
--- a/src/allmydata/test/cli/test_mv.py
+++ b/src/allmydata/test/cli/test_mv.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 from twisted.trial import unittest
 from allmydata.util import fileutil
@@ -5,6 +17,7 @@ from ..no_network import GridTestMixin
 from allmydata.scripts import tahoe_mv
 from .common import CLITestMixin
 
+
 class Mv(GridTestMixin, CLITestMixin, unittest.TestCase):
     def test_mv_behavior(self):
         self.basedir = "cli/Mv/mv_behavior"
@@ -152,7 +165,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         # check to see that the validation extends to the
         # target argument by making an alias that will work with the first
@@ -180,7 +193,7 @@ class Mv(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("fake", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(len(out), 0, out)
         d.addCallback(_check)
         # check to see that the validation extends to the
         # target argument by making an alias that will work with the first
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 400a0ea9e..919192452 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -182,6 +182,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_create_alias",
     "allmydata.test.cli.test_invite",
     "allmydata.test.cli.test_list",
+    "allmydata.test.cli.test_mv",
     "allmydata.test.cli.test_put",
     "allmydata.test.cli.test_status",
 

From 02edef01a9e28ee853ce844761596946deb352e9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:15:24 -0400
Subject: [PATCH 66/81] News file.

---
 newsfragments/3691.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3691.minor

diff --git a/newsfragments/3691.minor b/newsfragments/3691.minor
new file mode 100644
index 000000000..e69de29bb

From f9ae91a94ec5b078c48440be047ab318a159f015 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:20:16 -0400
Subject: [PATCH 67/81] Tests pass on Python 3.

---
 src/allmydata/test/cli/test_run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index d27791f34..0e12483dd 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -50,7 +50,7 @@ class DaemonizeTheRealServiceTests(SyncTestCase):
         """
         nodedir = FilePath(self.mktemp())
         nodedir.makedirs()
-        nodedir.child("tahoe.cfg").setContent(config)
+        nodedir.child("tahoe.cfg").setContent(config.encode("ascii"))
         nodedir.child("tahoe-client.tac").touch()
 
         options = parse_options(["run", nodedir.path])

From 1f70d5c13a5d094a32f2f38fb699fecfefdbc40e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:21:23 -0400
Subject: [PATCH 68/81] Port to Python 3.

---
 src/allmydata/test/cli/test_run.py | 10 ++++++++++
 src/allmydata/util/_python3.py     |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index 0e12483dd..6100d2568 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -1,6 +1,16 @@
 """
 Tests for ``allmydata.scripts.tahoe_run``.
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from six.moves import (
     StringIO,
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 919192452..bbafc144f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -184,6 +184,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_list",
     "allmydata.test.cli.test_mv",
     "allmydata.test.cli.test_put",
+    "allmydata.test.cli.test_run",
     "allmydata.test.cli.test_status",
 
     "allmydata.test.mutable.test_checker",

From d3be3ce1e6978976dc800a98b80c4af27d600c3b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:43:31 -0400
Subject: [PATCH 69/81] Start making tests pass on Python 3.

---
 src/allmydata/scripts/admin.py     | 15 ++++++++++-----
 src/allmydata/test/cli/test_cli.py | 11 +++++++----
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py
index 50dde9e43..0c860e93a 100644
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 try:
     from allmydata.scripts.types_ import SubCommands
 except ImportError:
@@ -22,8 +24,10 @@ def print_keypair(options):
     from allmydata.crypto import ed25519
     out = options.stdout
     private_key, public_key = ed25519.create_signing_keypair()
-    print("private:", ed25519.string_from_signing_key(private_key), file=out)
-    print("public:", ed25519.string_from_verifying_key(public_key), file=out)
+    print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"),
+          file=out)
+    print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"),
+          file=out)
 
 class DerivePubkeyOptions(BaseOptions):
     def parseArgs(self, privkey):
@@ -45,9 +49,10 @@ def derive_pubkey(options):
     out = options.stdout
     from allmydata.crypto import ed25519
     privkey_vs = options.privkey
-    private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs)
-    print("private:", ed25519.string_from_signing_key(private_key), file=out)
-    print("public:", ed25519.string_from_verifying_key(public_key), file=out)
+    private_key, public_key = ed25519.signing_keypair_from_string(
+        privkey_vs.encode("ascii"))
+    print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out)
+    print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out)
     return 0
 
 class AdminCommand(BaseOptions):
diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index 2b1bc1c86..2e3c4d86a 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -1,3 +1,5 @@
+from past.builtins import unicode
+
 import os.path
 from six.moves import cStringIO as StringIO
 import urllib, sys
@@ -718,8 +720,9 @@ class Admin(unittest.TestCase):
             self.failUnlessEqual(pubkey_bits[0], vk_header, lines[1])
             self.failUnless(privkey_bits[1].startswith("priv-v0-"), lines[0])
             self.failUnless(pubkey_bits[1].startswith("pub-v0-"), lines[1])
-            sk, pk = ed25519.signing_keypair_from_string(privkey_bits[1])
-            vk_bytes = pubkey_bits[1]
+            sk, pk = ed25519.signing_keypair_from_string(
+                privkey_bits[1].encode("ascii"))
+            vk_bytes = pubkey_bits[1].encode("ascii")
             self.assertEqual(
                 ed25519.string_from_verifying_key(pk),
                 vk_bytes,
@@ -729,8 +732,8 @@ class Admin(unittest.TestCase):
 
     def test_derive_pubkey(self):
         priv_key, pub_key = ed25519.create_signing_keypair()
-        priv_key_str = ed25519.string_from_signing_key(priv_key)
-        pub_key_str = ed25519.string_from_verifying_key(pub_key)
+        priv_key_str = unicode(ed25519.string_from_signing_key(priv_key), "ascii")
+        pub_key_str = unicode(ed25519.string_from_verifying_key(pub_key), "ascii")
         d = run_cli("admin", "derive-pubkey", priv_key_str)
         def _done(args):
             (rc, stdout, stderr) = args

From ae739dfd9e1f81d4b5b62a1e0570b24305827c50 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 3 May 2021 11:48:02 -0400
Subject: [PATCH 70/81] Python 3 updates.

---
 src/allmydata/test/cli/test_cli.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index 2e3c4d86a..d08d46a34 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -363,11 +363,11 @@ class CLI(CLITestMixin, unittest.TestCase):
                         "didn't see 'mqfblse6m5a6dh45isu2cg7oji' in '%s'" % err)
 
     def test_alias(self):
-        def s128(c): return base32.b2a(c*(128/8))
-        def s256(c): return base32.b2a(c*(256/8))
-        TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T"))
-        WA = "URI:DIR2:%s:%s" % (s128("W"), s256("W"))
-        CA = "URI:DIR2:%s:%s" % (s128("C"), s256("C"))
+        def s128(c): return base32.b2a(c*(128//8))
+        def s256(c): return base32.b2a(c*(256//8))
+        TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T"))
+        WA = b"URI:DIR2:%s:%s" % (s128(b"W"), s256(b"W"))
+        CA = b"URI:DIR2:%s:%s" % (s128(b"C"), s256(b"C"))
         aliases = {"tahoe": TA,
                    "work": WA,
                    "c": CA}

From 7349855ce4a417fcce23f9192354e733803f4bd1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:00:27 -0400
Subject: [PATCH 71/81] Move unicode conversion higher up.

---
 src/allmydata/scripts/tahoe_ls.py | 3 ++-
 src/allmydata/scripts/tahoe_mv.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_ls.py b/src/allmydata/scripts/tahoe_ls.py
index 92d5adfef..b3e09b699 100644
--- a/src/allmydata/scripts/tahoe_ls.py
+++ b/src/allmydata/scripts/tahoe_ls.py
@@ -27,6 +27,8 @@ def list(options):
     except UnknownAliasError as e:
         e.display(stderr)
         return 1
+
+    path = unicode(path, "utf-8")
     url = nodeurl + "uri/%s" % url_quote(rootcap)
     if path:
         # move where.endswith check here?
@@ -64,7 +66,6 @@ def list(options):
         print(quote_output(data, quotemarks=False), file=stderr)
         return 1
 
-    path = unicode(path, "utf-8")
     nodetype, d = parsed
     children = {}
     if nodetype == "dirnode":
diff --git a/src/allmydata/scripts/tahoe_mv.py b/src/allmydata/scripts/tahoe_mv.py
index 580475c91..84f83edcd 100644
--- a/src/allmydata/scripts/tahoe_mv.py
+++ b/src/allmydata/scripts/tahoe_mv.py
@@ -27,6 +27,7 @@ def mv(options, mode="move"):
     except UnknownAliasError as e:
         e.display(stderr)
         return 1
+    from_path = unicode(from_path, "utf-8")
     from_url = nodeurl + "uri/%s" % url_quote(rootcap)
     if from_path:
         from_url += "/" + escape_path(from_path)
@@ -46,10 +47,10 @@ def mv(options, mode="move"):
         e.display(stderr)
         return 1
     to_url = nodeurl + "uri/%s" % url_quote(rootcap)
+    path = unicode(path, "utf-8")
     if path:
         to_url += "/" + escape_path(path)
 
-    from_path = unicode(from_path, "utf-8")
     if to_url.endswith("/"):
         # "mv foo.txt bar/" == "mv foo.txt bar/foo.txt"
         to_url += escape_path(from_path[from_path.rfind("/")+1:])

From 91f8575d29a64c8702095c22865e738da3e775dd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:04:45 -0400
Subject: [PATCH 72/81] News file

---
 newsfragments/3692.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3692.minor

diff --git a/newsfragments/3692.minor b/newsfragments/3692.minor
new file mode 100644
index 000000000..e69de29bb

From 75deef906d849255d0ffd9b1efadc7121c3ad2d9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:12:07 -0400
Subject: [PATCH 73/81] More progress towards running tests on Python 3.

---
 src/allmydata/test/cli/test_cli.py | 129 +++++++++++++++--------------
 1 file changed, 65 insertions(+), 64 deletions(-)

diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index d08d46a34..405340115 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -2,9 +2,10 @@ from past.builtins import unicode
 
 import os.path
 from six.moves import cStringIO as StringIO
-import urllib, sys
+import sys
 import re
 from mock import patch, Mock
+from urllib.parse import quote as url_quote
 
 from twisted.trial import unittest
 from twisted.python.monkey import MonkeyPatcher
@@ -55,8 +56,8 @@ class CLI(CLITestMixin, unittest.TestCase):
         return output
 
     def test_dump_cap_chk(self):
-        key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-        uri_extension_hash = hashutil.uri_extension_hash("stuff")
+        key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+        uri_extension_hash = hashutil.uri_extension_hash(b"stuff")
         needed_shares = 25
         total_shares = 100
         size = 1234
@@ -65,7 +66,7 @@ class CLI(CLITestMixin, unittest.TestCase):
                            needed_shares=needed_shares,
                            total_shares=total_shares,
                            size=size)
-        output = self._dump_cap(u.to_string())
+        output = self._dump_cap(unicode(u.to_string(), "ascii"))
         self.failUnless("CHK File:" in output, output)
         self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output)
         self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output)
@@ -84,7 +85,7 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnless("k/N: 25/100" in output, output)
         self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output)
 
-        prefixed_u = "http://127.0.0.1/uri/%s" % urllib.quote(u.to_string())
+        prefixed_u = "http://127.0.0.1/uri/%s" % url_quote(u.to_string())
         output = self._dump_cap(prefixed_u)
         self.failUnless("CHK File:" in output, output)
         self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output)
@@ -374,60 +375,60 @@ class CLI(CLITestMixin, unittest.TestCase):
         def ga1(path):
             return get_alias(aliases, path, u"tahoe")
         uses_lettercolon = common.platform_uses_lettercolon_drivename()
-        self.failUnlessReallyEqual(ga1(u"bare"), (TA, "bare"))
-        self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, "baredir/file"))
-        self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, "baredir/file:7"))
-        self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, ""))
-        self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, "file"))
-        self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, "dir/file"))
-        self.failUnlessReallyEqual(ga1(u"work:"), (WA, ""))
-        self.failUnlessReallyEqual(ga1(u"work:file"), (WA, "file"))
-        self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, "dir/file"))
+        self.failUnlessReallyEqual(ga1(u"bare"), (TA, b"bare"))
+        self.failUnlessReallyEqual(ga1(u"baredir/file"), (TA, b"baredir/file"))
+        self.failUnlessReallyEqual(ga1(u"baredir/file:7"), (TA, b"baredir/file:7"))
+        self.failUnlessReallyEqual(ga1(u"tahoe:"), (TA, b""))
+        self.failUnlessReallyEqual(ga1(u"tahoe:file"), (TA, b"file"))
+        self.failUnlessReallyEqual(ga1(u"tahoe:dir/file"), (TA, b"dir/file"))
+        self.failUnlessReallyEqual(ga1(u"work:"), (WA, b""))
+        self.failUnlessReallyEqual(ga1(u"work:file"), (WA, b"file"))
+        self.failUnlessReallyEqual(ga1(u"work:dir/file"), (WA, b"dir/file"))
         # default != None means we really expect a tahoe path, regardless of
         # whether we're on windows or not. This is what 'tahoe get' uses.
-        self.failUnlessReallyEqual(ga1(u"c:"), (CA, ""))
-        self.failUnlessReallyEqual(ga1(u"c:file"), (CA, "file"))
-        self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, "dir/file"))
-        self.failUnlessReallyEqual(ga1(u"URI:stuff"), ("URI:stuff", ""))
-        self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), ("URI:stuff", "file"))
-        self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), ("URI:stuff", "file"))
-        self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file"))
-        self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file"))
+        self.failUnlessReallyEqual(ga1(u"c:"), (CA, b""))
+        self.failUnlessReallyEqual(ga1(u"c:file"), (CA, b"file"))
+        self.failUnlessReallyEqual(ga1(u"c:dir/file"), (CA, b"dir/file"))
+        self.failUnlessReallyEqual(ga1(u"URI:stuff"), (b"URI:stuff", b""))
+        self.failUnlessReallyEqual(ga1(u"URI:stuff/file"), (b"URI:stuff", b"file"))
+        self.failUnlessReallyEqual(ga1(u"URI:stuff:./file"), (b"URI:stuff", b"file"))
+        self.failUnlessReallyEqual(ga1(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file"))
+        self.failUnlessReallyEqual(ga1(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file"))
         self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:")
         self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir")
         self.failUnlessRaises(common.UnknownAliasError, ga1, u"missing:dir/file")
 
         def ga2(path):
             return get_alias(aliases, path, None)
-        self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, "bare"))
+        self.failUnlessReallyEqual(ga2(u"bare"), (DefaultAliasMarker, b"bare"))
         self.failUnlessReallyEqual(ga2(u"baredir/file"),
-                             (DefaultAliasMarker, "baredir/file"))
+                             (DefaultAliasMarker, b"baredir/file"))
         self.failUnlessReallyEqual(ga2(u"baredir/file:7"),
-                             (DefaultAliasMarker, "baredir/file:7"))
+                             (DefaultAliasMarker, b"baredir/file:7"))
         self.failUnlessReallyEqual(ga2(u"baredir/sub:1/file:7"),
-                             (DefaultAliasMarker, "baredir/sub:1/file:7"))
-        self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, ""))
-        self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, "file"))
-        self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, "dir/file"))
+                             (DefaultAliasMarker, b"baredir/sub:1/file:7"))
+        self.failUnlessReallyEqual(ga2(u"tahoe:"), (TA, b""))
+        self.failUnlessReallyEqual(ga2(u"tahoe:file"), (TA, b"file"))
+        self.failUnlessReallyEqual(ga2(u"tahoe:dir/file"), (TA, b"dir/file"))
         # on windows, we really want c:foo to indicate a local file.
         # default==None is what 'tahoe cp' uses.
         if uses_lettercolon:
-            self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, "c:"))
-            self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, "c:file"))
+            self.failUnlessReallyEqual(ga2(u"c:"), (DefaultAliasMarker, b"c:"))
+            self.failUnlessReallyEqual(ga2(u"c:file"), (DefaultAliasMarker, b"c:file"))
             self.failUnlessReallyEqual(ga2(u"c:dir/file"),
-                                 (DefaultAliasMarker, "c:dir/file"))
+                                 (DefaultAliasMarker, b"c:dir/file"))
         else:
-            self.failUnlessReallyEqual(ga2(u"c:"), (CA, ""))
-            self.failUnlessReallyEqual(ga2(u"c:file"), (CA, "file"))
-            self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, "dir/file"))
-        self.failUnlessReallyEqual(ga2(u"work:"), (WA, ""))
-        self.failUnlessReallyEqual(ga2(u"work:file"), (WA, "file"))
-        self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, "dir/file"))
-        self.failUnlessReallyEqual(ga2(u"URI:stuff"), ("URI:stuff", ""))
-        self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), ("URI:stuff", "file"))
-        self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), ("URI:stuff", "file"))
-        self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), ("URI:stuff", "dir/file"))
-        self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file"))
+            self.failUnlessReallyEqual(ga2(u"c:"), (CA, b""))
+            self.failUnlessReallyEqual(ga2(u"c:file"), (CA, b"file"))
+            self.failUnlessReallyEqual(ga2(u"c:dir/file"), (CA, b"dir/file"))
+        self.failUnlessReallyEqual(ga2(u"work:"), (WA, b""))
+        self.failUnlessReallyEqual(ga2(u"work:file"), (WA, b"file"))
+        self.failUnlessReallyEqual(ga2(u"work:dir/file"), (WA, b"dir/file"))
+        self.failUnlessReallyEqual(ga2(u"URI:stuff"), (b"URI:stuff", b""))
+        self.failUnlessReallyEqual(ga2(u"URI:stuff/file"), (b"URI:stuff", b"file"))
+        self.failUnlessReallyEqual(ga2(u"URI:stuff:./file"), (b"URI:stuff", b"file"))
+        self.failUnlessReallyEqual(ga2(u"URI:stuff/dir/file"), (b"URI:stuff", b"dir/file"))
+        self.failUnlessReallyEqual(ga2(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file"))
         self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:")
         self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir")
         self.failUnlessRaises(common.UnknownAliasError, ga2, u"missing:dir/file")
@@ -440,26 +441,26 @@ class CLI(CLITestMixin, unittest.TestCase):
             finally:
                 common.pretend_platform_uses_lettercolon = old
             return retval
-        self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, "bare"))
+        self.failUnlessReallyEqual(ga3(u"bare"), (DefaultAliasMarker, b"bare"))
         self.failUnlessReallyEqual(ga3(u"baredir/file"),
-                             (DefaultAliasMarker, "baredir/file"))
+                             (DefaultAliasMarker, b"baredir/file"))
         self.failUnlessReallyEqual(ga3(u"baredir/file:7"),
-                             (DefaultAliasMarker, "baredir/file:7"))
+                             (DefaultAliasMarker, b"baredir/file:7"))
         self.failUnlessReallyEqual(ga3(u"baredir/sub:1/file:7"),
-                             (DefaultAliasMarker, "baredir/sub:1/file:7"))
-        self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, ""))
-        self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, "file"))
-        self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, "dir/file"))
-        self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, "c:"))
-        self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, "c:file"))
+                             (DefaultAliasMarker, b"baredir/sub:1/file:7"))
+        self.failUnlessReallyEqual(ga3(u"tahoe:"), (TA, b""))
+        self.failUnlessReallyEqual(ga3(u"tahoe:file"), (TA, b"file"))
+        self.failUnlessReallyEqual(ga3(u"tahoe:dir/file"), (TA, b"dir/file"))
+        self.failUnlessReallyEqual(ga3(u"c:"), (DefaultAliasMarker, b"c:"))
+        self.failUnlessReallyEqual(ga3(u"c:file"), (DefaultAliasMarker, b"c:file"))
         self.failUnlessReallyEqual(ga3(u"c:dir/file"),
-                             (DefaultAliasMarker, "c:dir/file"))
-        self.failUnlessReallyEqual(ga3(u"work:"), (WA, ""))
-        self.failUnlessReallyEqual(ga3(u"work:file"), (WA, "file"))
-        self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, "dir/file"))
-        self.failUnlessReallyEqual(ga3(u"URI:stuff"), ("URI:stuff", ""))
-        self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), ("URI:stuff", "file"))
-        self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), ("URI:stuff", "dir/file"))
+                             (DefaultAliasMarker, b"c:dir/file"))
+        self.failUnlessReallyEqual(ga3(u"work:"), (WA, b""))
+        self.failUnlessReallyEqual(ga3(u"work:file"), (WA, b"file"))
+        self.failUnlessReallyEqual(ga3(u"work:dir/file"), (WA, b"dir/file"))
+        self.failUnlessReallyEqual(ga3(u"URI:stuff"), (b"URI:stuff", b""))
+        self.failUnlessReallyEqual(ga3(u"URI:stuff:./file"), (b"URI:stuff", b"file"))
+        self.failUnlessReallyEqual(ga3(u"URI:stuff:./dir/file"), (b"URI:stuff", b"dir/file"))
         self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:")
         self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir")
         self.failUnlessRaises(common.UnknownAliasError, ga3, u"missing:dir/file")
@@ -482,14 +483,14 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnlessRaises(common.UnknownAliasError, ga5, u"C:\\Windows")
 
     def test_alias_tolerance(self):
-        def s128(c): return base32.b2a(c*(128/8))
-        def s256(c): return base32.b2a(c*(256/8))
-        TA = "URI:DIR2:%s:%s" % (s128("T"), s256("T"))
+        def s128(c): return base32.b2a(c*(128//8))
+        def s256(c): return base32.b2a(c*(256//8))
+        TA = b"URI:DIR2:%s:%s" % (s128(b"T"), s256(b"T"))
         aliases = {"present": TA,
-                   "future": "URI-FROM-FUTURE:ooh:aah"}
+                   "future": b"URI-FROM-FUTURE:ooh:aah"}
         def ga1(path):
             return get_alias(aliases, path, u"tahoe")
-        self.failUnlessReallyEqual(ga1(u"present:file"), (TA, "file"))
+        self.failUnlessReallyEqual(ga1(u"present:file"), (TA, b"file"))
         # this throws, via assert IDirnodeURI.providedBy(), since get_alias()
         # wants a dirnode, and the future cap gives us UnknownURI instead.
         self.failUnlessRaises(AssertionError, ga1, u"future:stuff")

From deaaa8c727fff53ca4e4f9c1b389feabd5b75108 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:27:26 -0400
Subject: [PATCH 74/81] More tests passing on Python 3.

---
 src/allmydata/scripts/debug.py     | 72 +++++++++++++++---------------
 src/allmydata/test/cli/test_cli.py | 30 +++++++------
 2 files changed, 52 insertions(+), 50 deletions(-)

diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py
index b8aeee91e..ba40519de 100644
--- a/src/allmydata/scripts/debug.py
+++ b/src/allmydata/scripts/debug.py
@@ -452,7 +452,7 @@ def dump_cap(options):
     from allmydata import uri
     from allmydata.util import base32
     from base64 import b32decode
-    import urlparse, urllib
+    from urllib.parse import unquote, urlparse
 
     out = options.stdout
     cap = options.cap
@@ -461,18 +461,18 @@ def dump_cap(options):
         nodeid = b32decode(options['nodeid'].upper())
     secret = None
     if options['client-secret']:
-        secret = base32.a2b(options['client-secret'])
+        secret = base32.a2b(options['client-secret'].encode("ascii"))
     elif options['client-dir']:
         secretfile = os.path.join(options['client-dir'], "private", "secret")
         try:
-            secret = base32.a2b(open(secretfile, "r").read().strip())
+            secret = base32.a2b(open(secretfile, "rb").read().strip())
         except EnvironmentError:
             pass
 
     if cap.startswith("http"):
-        scheme, netloc, path, params, query, fragment = urlparse.urlparse(cap)
+        scheme, netloc, path, params, query, fragment = urlparse(cap)
         assert path.startswith("/uri/")
-        cap = urllib.unquote(path[len("/uri/"):])
+        cap = unquote(path[len("/uri/"):])
 
     u = uri.from_string(cap)
 
@@ -485,19 +485,19 @@ def _dump_secrets(storage_index, secret, nodeid, out):
 
     if secret:
         crs = hashutil.my_renewal_secret_hash(secret)
-        print(" client renewal secret:", base32.b2a(crs), file=out)
+        print(" client renewal secret:", unicode(base32.b2a(crs), "ascii"), file=out)
         frs = hashutil.file_renewal_secret_hash(crs, storage_index)
-        print(" file renewal secret:", base32.b2a(frs), file=out)
+        print(" file renewal secret:", unicode(base32.b2a(frs), "ascii"), file=out)
         if nodeid:
             renew = hashutil.bucket_renewal_secret_hash(frs, nodeid)
-            print(" lease renewal secret:", base32.b2a(renew), file=out)
+            print(" lease renewal secret:", unicode(base32.b2a(renew), "ascii"), file=out)
         ccs = hashutil.my_cancel_secret_hash(secret)
-        print(" client cancel secret:", base32.b2a(ccs), file=out)
+        print(" client cancel secret:", unicode(base32.b2a(ccs), "ascii"), file=out)
         fcs = hashutil.file_cancel_secret_hash(ccs, storage_index)
-        print(" file cancel secret:", base32.b2a(fcs), file=out)
+        print(" file cancel secret:", unicode(base32.b2a(fcs), "ascii"), file=out)
         if nodeid:
             cancel = hashutil.bucket_cancel_secret_hash(fcs, nodeid)
-            print(" lease cancel secret:", base32.b2a(cancel), file=out)
+            print(" lease cancel secret:", unicode(base32.b2a(cancel), "ascii"), file=out)
 
 def dump_uri_instance(u, nodeid, secret, out, show_header=True):
     from allmydata import uri
@@ -508,19 +508,19 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True):
     if isinstance(u, uri.CHKFileURI):
         if show_header:
             print("CHK File:", file=out)
-        print(" key:", base32.b2a(u.key), file=out)
-        print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out)
+        print(" key:", unicode(base32.b2a(u.key), "ascii"), file=out)
+        print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
         print(" size:", u.size, file=out)
         print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
         _dump_secrets(u.get_storage_index(), secret, nodeid, out)
     elif isinstance(u, uri.CHKFileVerifierURI):
         if show_header:
             print("CHK Verifier URI:", file=out)
-        print(" UEB hash:", base32.b2a(u.uri_extension_hash), file=out)
+        print(" UEB hash:", unicode(base32.b2a(u.uri_extension_hash), "ascii"), file=out)
         print(" size:", u.size, file=out)
         print(" k/N: %d/%d" % (u.needed_shares, u.total_shares), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
 
     elif isinstance(u, uri.LiteralFileURI):
         if show_header:
@@ -530,52 +530,52 @@ def dump_uri_instance(u, nodeid, secret, out, show_header=True):
     elif isinstance(u, uri.WriteableSSKFileURI): # SDMF
         if show_header:
             print("SDMF Writeable URI:", file=out)
-        print(" writekey:", base32.b2a(u.writekey), file=out)
-        print(" readkey:", base32.b2a(u.readkey), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out)
+        print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
         print(file=out)
         if nodeid:
             we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
-            print(" write_enabler:", base32.b2a(we), file=out)
+            print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out)
             print(file=out)
         _dump_secrets(u.get_storage_index(), secret, nodeid, out)
     elif isinstance(u, uri.ReadonlySSKFileURI):
         if show_header:
             print("SDMF Read-only URI:", file=out)
-        print(" readkey:", base32.b2a(u.readkey), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
     elif isinstance(u, uri.SSKVerifierURI):
         if show_header:
             print("SDMF Verifier URI:", file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
 
     elif isinstance(u, uri.WriteableMDMFFileURI): # MDMF
         if show_header:
             print("MDMF Writeable URI:", file=out)
-        print(" writekey:", base32.b2a(u.writekey), file=out)
-        print(" readkey:", base32.b2a(u.readkey), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" writekey:", unicode(base32.b2a(u.writekey), "ascii"), file=out)
+        print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
         print(file=out)
         if nodeid:
             we = hashutil.ssk_write_enabler_hash(u.writekey, nodeid)
-            print(" write_enabler:", base32.b2a(we), file=out)
+            print(" write_enabler:", unicode(base32.b2a(we), "ascii"), file=out)
             print(file=out)
         _dump_secrets(u.get_storage_index(), secret, nodeid, out)
     elif isinstance(u, uri.ReadonlyMDMFFileURI):
         if show_header:
             print("MDMF Read-only URI:", file=out)
-        print(" readkey:", base32.b2a(u.readkey), file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" readkey:", unicode(base32.b2a(u.readkey), "ascii"), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
     elif isinstance(u, uri.MDMFVerifierURI):
         if show_header:
             print("MDMF Verifier URI:", file=out)
-        print(" storage index:", si_b2a(u.get_storage_index()), file=out)
-        print(" fingerprint:", base32.b2a(u.fingerprint), file=out)
+        print(" storage index:", unicode(si_b2a(u.get_storage_index()), "ascii"), file=out)
+        print(" fingerprint:", unicode(base32.b2a(u.fingerprint), "ascii"), file=out)
 
 
     elif isinstance(u, uri.ImmutableDirectoryURI): # CHK-based directory
diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index 405340115..d1fc48899 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -47,6 +47,8 @@ from allmydata.util.encodingutil import listdir_unicode, get_io_encoding
 
 class CLI(CLITestMixin, unittest.TestCase):
     def _dump_cap(self, *args):
+        args = [(unicode(s, "ascii") if isinstance(s, bytes) else s)
+                for s in args]
         config = debug.DumpCapOptions()
         config.stdout,config.stderr = StringIO(), StringIO()
         config.parseOptions(args)
@@ -66,7 +68,7 @@ class CLI(CLITestMixin, unittest.TestCase):
                            needed_shares=needed_shares,
                            total_shares=total_shares,
                            size=size)
-        output = self._dump_cap(unicode(u.to_string(), "ascii"))
+        output = self._dump_cap(u.to_string())
         self.failUnless("CHK File:" in output, output)
         self.failUnless("key: aaaqeayeaudaocajbifqydiob4" in output, output)
         self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output)
@@ -78,7 +80,7 @@ class CLI(CLITestMixin, unittest.TestCase):
                                 u.to_string())
         self.failUnless("client renewal secret: znxmki5zdibb5qlt46xbdvk2t55j7hibejq3i5ijyurkr6m6jkhq" in output, output)
 
-        output = self._dump_cap(u.get_verify_cap().to_string())
+        output = self._dump_cap(unicode(u.get_verify_cap().to_string(), "ascii"))
         self.failIf("key: " in output, output)
         self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output)
         self.failUnless("size: 1234" in output, output)
@@ -95,14 +97,14 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output)
 
     def test_dump_cap_lit(self):
-        u = uri.LiteralFileURI("this is some data")
+        u = uri.LiteralFileURI(b"this is some data")
         output = self._dump_cap(u.to_string())
         self.failUnless("Literal File URI:" in output, output)
         self.failUnless("data: 'this is some data'" in output, output)
 
     def test_dump_cap_sdmf(self):
-        writekey = "\x01" * 16
-        fingerprint = "\xfe" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\xfe" * 32
         u = uri.WriteableSSKFileURI(writekey, fingerprint)
 
         output = self._dump_cap(u.to_string())
@@ -152,8 +154,8 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output)
 
     def test_dump_cap_mdmf(self):
-        writekey = "\x01" * 16
-        fingerprint = "\xfe" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\xfe" * 32
         u = uri.WriteableMDMFFileURI(writekey, fingerprint)
 
         output = self._dump_cap(u.to_string())
@@ -204,8 +206,8 @@ class CLI(CLITestMixin, unittest.TestCase):
 
 
     def test_dump_cap_chk_directory(self):
-        key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-        uri_extension_hash = hashutil.uri_extension_hash("stuff")
+        key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+        uri_extension_hash = hashutil.uri_extension_hash(b"stuff")
         needed_shares = 25
         total_shares = 100
         size = 1234
@@ -238,8 +240,8 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnless("storage index: hdis5iaveku6lnlaiccydyid7q" in output, output)
 
     def test_dump_cap_sdmf_directory(self):
-        writekey = "\x01" * 16
-        fingerprint = "\xfe" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\xfe" * 32
         u1 = uri.WriteableSSKFileURI(writekey, fingerprint)
         u = uri.DirectoryURI(u1)
 
@@ -282,8 +284,8 @@ class CLI(CLITestMixin, unittest.TestCase):
         self.failUnless("fingerprint: 737p57x6737p57x6737p57x6737p57x6737p57x6737p57x6737a" in output, output)
 
     def test_dump_cap_mdmf_directory(self):
-        writekey = "\x01" * 16
-        fingerprint = "\xfe" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\xfe" * 32
         u1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
         u = uri.MDMFDirectoryURI(u1)
 
@@ -343,7 +345,7 @@ class CLI(CLITestMixin, unittest.TestCase):
         fileutil.write("cli/test_catalog_shares/node1/storage/shares/mq/not-a-dir", "")
         # write a bogus share that looks a little bit like CHK
         fileutil.write(os.path.join(sharedir, "8"),
-                       "\x00\x00\x00\x01" + "\xff" * 200) # this triggers an assert
+                       b"\x00\x00\x00\x01" + b"\xff" * 200) # this triggers an assert
 
         nodedir2 = "cli/test_catalog_shares/node2"
         fileutil.make_dirs(nodedir2)

From a4af4d8e5bb3b21a6e0c37290678bb78a6f2db51 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:36:27 -0400
Subject: [PATCH 75/81] Even more passing tests on Python 3.

---
 src/allmydata/scripts/cli.py       |  4 +++-
 src/allmydata/test/cli/common.py   |  3 ++-
 src/allmydata/test/cli/test_cli.py | 33 +++++++++++++++---------------
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py
index 645b57f41..011dc3b21 100644
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 import os.path, re, fnmatch
 
 try:
@@ -36,7 +38,7 @@ class FileStoreOptions(BaseOptions):
 
         # compute a node-url from the existing options, put in self['node-url']
         if self['node-url']:
-            if (not isinstance(self['node-url'], basestring)
+            if (not isinstance(self['node-url'], (bytes, unicode))
                 or not NODEURL_RE.match(self['node-url'])):
                 msg = ("--node-url is required to be a string and look like "
                        "\"http://HOSTNAMEORADDR:PORT\", not: %r" %
diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py
index 8796f815f..7b97312b0 100644
--- a/src/allmydata/test/cli/common.py
+++ b/src/allmydata/test/cli/common.py
@@ -1,9 +1,10 @@
-from six import ensure_str
+from six import ensure_str, ensure_text
 
 from ...scripts import runner
 from ..common_util import ReallyEqualMixin, run_cli, run_cli_unicode
 
 def parse_options(basedir, command, args):
+    args = [ensure_text(s) for s in args]
     o = runner.Options()
     o.parseOptions(["--node-directory", basedir, command] + args)
     while hasattr(o, "subOptions"):
diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index d1fc48899..85a3ea11e 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -1,7 +1,9 @@
 from past.builtins import unicode
 
-import os.path
 from six.moves import cStringIO as StringIO
+from six import ensure_text
+
+import os.path
 import sys
 import re
 from mock import patch, Mock
@@ -47,8 +49,7 @@ from allmydata.util.encodingutil import listdir_unicode, get_io_encoding
 
 class CLI(CLITestMixin, unittest.TestCase):
     def _dump_cap(self, *args):
-        args = [(unicode(s, "ascii") if isinstance(s, bytes) else s)
-                for s in args]
+        args = [ensure_text(s) for s in args]
         config = debug.DumpCapOptions()
         config.stdout,config.stderr = StringIO(), StringIO()
         config.parseOptions(args)
@@ -759,8 +760,8 @@ class Errors(GridTestMixin, CLITestMixin, unittest.TestCase):
         self.set_up_grid()
         c0 = self.g.clients[0]
         self.fileurls = {}
-        DATA = "data" * 100
-        d = c0.upload(upload.Data(DATA, convergence=""))
+        DATA = b"data" * 100
+        d = c0.upload(upload.Data(DATA, convergence=b""))
         def _stash_bad(ur):
             self.uri_1share = ur.get_uri()
             self.delete_shares_numbered(ur.get_uri(), range(1,10))
@@ -1201,31 +1202,31 @@ class Options(ReallyEqualMixin, unittest.TestCase):
         fileutil.make_dirs("cli/test_options")
         fileutil.make_dirs("cli/test_options/private")
         fileutil.write("cli/test_options/node.url", "http://localhost:8080/\n")
-        filenode_uri = uri.WriteableSSKFileURI(writekey="\x00"*16,
-                                               fingerprint="\x00"*32)
+        filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x00"*16,
+                                               fingerprint=b"\x00"*32)
         private_uri = uri.DirectoryURI(filenode_uri).to_string()
-        fileutil.write("cli/test_options/private/root_dir.cap", private_uri + "\n")
+        fileutil.write("cli/test_options/private/root_dir.cap", private_uri + b"\n")
         def parse2(args): return parse_options("cli/test_options", "ls", args)
         o = parse2([])
         self.failUnlessEqual(o['node-url'], "http://localhost:8080/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri)
         self.failUnlessEqual(o.where, u"")
 
         o = parse2(["--node-url", "http://example.org:8111/"])
         self.failUnlessEqual(o['node-url'], "http://example.org:8111/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri)
         self.failUnlessEqual(o.where, u"")
 
         # -u for --node-url used to clash with -u for --uri (tickets #1949 and #2137).
         o = parse2(["-u", "http://example.org:8111/"])
         self.failUnlessEqual(o['node-url'], "http://example.org:8111/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri)
         self.failUnlessEqual(o.where, u"")
         self.failIf(o["uri"])
 
         o = parse2(["-u", "http://example.org:8111/", "--uri"])
         self.failUnlessEqual(o['node-url'], "http://example.org:8111/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], private_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), private_uri)
         self.failUnlessEqual(o.where, u"")
         self.failUnless(o["uri"])
 
@@ -1234,17 +1235,17 @@ class Options(ReallyEqualMixin, unittest.TestCase):
         self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], "root")
         self.failUnlessEqual(o.where, u"")
 
-        other_filenode_uri = uri.WriteableSSKFileURI(writekey="\x11"*16,
-                                                     fingerprint="\x11"*32)
+        other_filenode_uri = uri.WriteableSSKFileURI(writekey=b"\x11"*16,
+                                                     fingerprint=b"\x11"*32)
         other_uri = uri.DirectoryURI(other_filenode_uri).to_string()
         o = parse2(["--dir-cap", other_uri])
         self.failUnlessEqual(o['node-url'], "http://localhost:8080/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri)
         self.failUnlessEqual(o.where, u"")
 
         o = parse2(["--dir-cap", other_uri, "subdir"])
         self.failUnlessEqual(o['node-url'], "http://localhost:8080/")
-        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS], other_uri)
+        self.failUnlessEqual(o.aliases[DEFAULT_ALIAS].encode("ascii"), other_uri)
         self.failUnlessEqual(o.where, u"subdir")
 
         self.failUnlessRaises(usage.UsageError, parse2,

From c589e97cdeff2c566e9664792700789898afa062 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:40:43 -0400
Subject: [PATCH 76/81] All tests pass on Python 3.

---
 src/allmydata/test/cli/test_cli.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index 85a3ea11e..3035bdb0e 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -1,7 +1,7 @@
 from past.builtins import unicode
 
 from six.moves import cStringIO as StringIO
-from six import ensure_text
+from six import ensure_text, ensure_str
 
 import os.path
 import sys
@@ -1151,7 +1151,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase):
         # TODO: replace with @patch that supports Deferreds.
         import webbrowser
         def call_webbrowser_open(url):
-            self.failUnlessIn(self.alias_uri.replace(':', '%3A'), url)
+            self.failUnlessIn(unicode(self.alias_uri, "ascii").replace(':', '%3A'), url)
             self.webbrowser_open_called = True
         def _cleanup(res):
             webbrowser.open = self.old_webbrowser_open
@@ -1332,7 +1332,7 @@ class Run(unittest.TestCase):
         If the pidfile exists but does not contain a numeric value, a complaint to
         this effect is written to stderr.
         """
-        basedir = FilePath(self.mktemp().decode("ascii"))
+        basedir = FilePath(ensure_str(self.mktemp()))
         basedir.makedirs()
         basedir.child(u"twistd.pid").setContent(b"foo")
         basedir.child(u"tahoe-client.tac").setContent(b"")
@@ -1340,7 +1340,7 @@ class Run(unittest.TestCase):
         config = tahoe_run.RunOptions()
         config.stdout = StringIO()
         config.stderr = StringIO()
-        config['basedir'] = basedir.path
+        config['basedir'] = ensure_text(basedir.path)
         config.twistd_args = []
 
         result_code = tahoe_run.run(config)

From 4c6d55b2600c61ceb9e736b6a8ae821917cd1104 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:52:55 -0400
Subject: [PATCH 77/81] Port to Python 3.

---
 src/allmydata/scripts/admin.py     |  5 ++-
 src/allmydata/test/cli/test_cli.py | 68 +++++++++++++++++-------------
 src/allmydata/util/_python3.py     |  1 +
 3 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py
index 0c860e93a..abe3d093c 100644
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 from past.builtins import unicode
+from six import ensure_binary
 
 try:
     from allmydata.scripts.types_ import SubCommands
@@ -49,8 +50,8 @@ def derive_pubkey(options):
     out = options.stdout
     from allmydata.crypto import ed25519
     privkey_vs = options.privkey
-    private_key, public_key = ed25519.signing_keypair_from_string(
-        privkey_vs.encode("ascii"))
+    privkey_vs = ensure_binary(privkey_vs)
+    private_key, public_key = ed25519.signing_keypair_from_string(privkey_vs)
     print("private:", unicode(ed25519.string_from_signing_key(private_key), "ascii"), file=out)
     print("public:", unicode(ed25519.string_from_verifying_key(public_key), "ascii"), file=out)
     return 0
diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py
index 3035bdb0e..8a9b4dfd6 100644
--- a/src/allmydata/test/cli/test_cli.py
+++ b/src/allmydata/test/cli/test_cli.py
@@ -1,4 +1,14 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from six.moves import cStringIO as StringIO
 from six import ensure_text, ensure_str
@@ -81,7 +91,7 @@ class CLI(CLITestMixin, unittest.TestCase):
                                 u.to_string())
         self.failUnless("client renewal secret: znxmki5zdibb5qlt46xbdvk2t55j7hibejq3i5ijyurkr6m6jkhq" in output, output)
 
-        output = self._dump_cap(unicode(u.get_verify_cap().to_string(), "ascii"))
+        output = self._dump_cap(str(u.get_verify_cap().to_string(), "ascii"))
         self.failIf("key: " in output, output)
         self.failUnless("UEB hash: nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa" in output, output)
         self.failUnless("size: 1234" in output, output)
@@ -354,7 +364,7 @@ class CLI(CLITestMixin, unittest.TestCase):
 
         # now make sure that the 'catalog-shares' commands survives the error
         out, err = self._catalog_shares(nodedir1, nodedir2)
-        self.failUnlessReallyEqual(out, "", out)
+        self.assertEqual(out, "")
         self.failUnless("Error processing " in err,
                         "didn't see 'error processing' in '%s'" % err)
         #self.failUnless(nodedir1 in err,
@@ -508,9 +518,9 @@ class CLI(CLITestMixin, unittest.TestCase):
         fileutil.make_dirs(basedir)
 
         for name in filenames:
-            open(os.path.join(unicode(basedir), name), "wb").close()
+            open(os.path.join(str(basedir), name), "wb").close()
 
-        for file in listdir_unicode(unicode(basedir)):
+        for file in listdir_unicode(str(basedir)):
             self.failUnlessIn(normalize(file), filenames)
 
     def test_exception_catcher(self):
@@ -677,7 +687,7 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         # Make sure that validation extends to the "to" parameter
         d.addCallback(lambda ign: self.do_cli("create-alias", "havasu"))
@@ -736,8 +746,8 @@ class Admin(unittest.TestCase):
 
     def test_derive_pubkey(self):
         priv_key, pub_key = ed25519.create_signing_keypair()
-        priv_key_str = unicode(ed25519.string_from_signing_key(priv_key), "ascii")
-        pub_key_str = unicode(ed25519.string_from_verifying_key(pub_key), "ascii")
+        priv_key_str = str(ed25519.string_from_signing_key(priv_key), "ascii")
+        pub_key_str = str(ed25519.string_from_verifying_key(pub_key), "ascii")
         d = run_cli("admin", "derive-pubkey", priv_key_str)
         def _done(args):
             (rc, stdout, stderr) = args
@@ -764,7 +774,7 @@ class Errors(GridTestMixin, CLITestMixin, unittest.TestCase):
         d = c0.upload(upload.Data(DATA, convergence=b""))
         def _stash_bad(ur):
             self.uri_1share = ur.get_uri()
-            self.delete_shares_numbered(ur.get_uri(), range(1,10))
+            self.delete_shares_numbered(ur.get_uri(), list(range(1,10)))
         d.addCallback(_stash_bad)
 
         # the download is abandoned as soon as it's clear that we won't get
@@ -828,7 +838,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -843,7 +853,7 @@ class Get(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("nonexistent", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -860,7 +870,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -875,7 +885,7 @@ class Manifest(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("nonexistent", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -890,7 +900,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(err, "")
             self.failUnlessIn("URI:", out)
         d.addCallback(_check)
 
@@ -903,7 +913,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check(args, st):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(err, "")
             self.failUnlessIn(st, out)
             return out
 
@@ -939,7 +949,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check(args, st):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(err, "")
             self.failUnlessIn(st, out)
             return out
         d.addCallback(_check, "URI:DIR2")
@@ -983,7 +993,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 0)
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(err, "")
             self.failUnlessIn("URI:", out)
         d.addCallback(_check)
 
@@ -999,7 +1009,7 @@ class Mkdir(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -1023,7 +1033,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
 
         d.addCallback(lambda ign: self.do_cli(self.command, "afile"))
@@ -1041,7 +1051,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
             self.failUnlessIn("nonexistent", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
 
         d.addCallback(lambda ign: self.do_cli(self.command, "nonexistent:afile"))
@@ -1067,7 +1077,7 @@ class Unlink(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("'tahoe %s'" % (self.command,), err)
             self.failUnlessIn("path must be given", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -1088,7 +1098,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(lambda ign: self.do_cli("stats", self.rooturi))
         def _check_stats(args):
             (rc, out, err) = args
-            self.failUnlessReallyEqual(err, "")
+            self.assertEqual(err, "")
             self.failUnlessReallyEqual(rc, 0)
             lines = out.splitlines()
             self.failUnlessIn(" count-immutable-files: 0", lines)
@@ -1112,7 +1122,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -1126,7 +1136,7 @@ class Stats(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -1143,7 +1153,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase):
             (rc, out, err) = args
             self.failUnlessReallyEqual(rc, 1)
             self.failUnlessIn("error:", err)
-            self.failUnlessReallyEqual(out, "")
+            self.assertEqual(out, "")
         d.addCallback(_check)
         return d
 
@@ -1151,7 +1161,7 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase):
         # TODO: replace with @patch that supports Deferreds.
         import webbrowser
         def call_webbrowser_open(url):
-            self.failUnlessIn(unicode(self.alias_uri, "ascii").replace(':', '%3A'), url)
+            self.failUnlessIn(str(self.alias_uri, "ascii").replace(':', '%3A'), url)
             self.webbrowser_open_called = True
         def _cleanup(res):
             webbrowser.open = self.old_webbrowser_open
@@ -1168,15 +1178,15 @@ class Webopen(GridTestMixin, CLITestMixin, unittest.TestCase):
                 (rc, out, err) = args
                 self.failUnlessReallyEqual(rc, 0, repr((rc, out, err)))
                 self.failUnlessIn("Alias 'alias' created", out)
-                self.failUnlessReallyEqual(err, "")
+                self.assertEqual(err, "")
                 self.alias_uri = get_aliases(self.get_clientdir())["alias"]
             d.addCallback(_check_alias)
             d.addCallback(lambda res: self.do_cli("webopen", "alias:"))
             def _check_webopen(args):
                 (rc, out, err) = args
                 self.failUnlessReallyEqual(rc, 0, repr((rc, out, err)))
-                self.failUnlessReallyEqual(out, "")
-                self.failUnlessReallyEqual(err, "")
+                self.assertEqual(out, "")
+                self.assertEqual(err, "")
                 self.failUnless(self.webbrowser_open_called)
             d.addCallback(_check_webopen)
             d.addBoth(_cleanup)
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index bbafc144f..f74222189 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -177,6 +177,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_backup",
     "allmydata.test.cli.test_backupdb",
     "allmydata.test.cli.test_check",
+    "allmydata.test.cli.test_cli",
     "allmydata.test.cli.test_cp",
     "allmydata.test.cli.test_create",
     "allmydata.test.cli.test_create_alias",

From 7411da1b88ed3ae69cc0ec7947f26d537667c43f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 10:57:45 -0400
Subject: [PATCH 78/81] Port to Python 3.

---
 src/allmydata/test/cli/common.py | 12 ++++++++++++
 src/allmydata/util/_python3.py   |  1 +
 2 files changed, 13 insertions(+)

diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py
index 7b97312b0..ed066c6b6 100644
--- a/src/allmydata/test/cli/common.py
+++ b/src/allmydata/test/cli/common.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from six import ensure_str, ensure_text
 
 from ...scripts import runner
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f74222189..d708a52ac 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -98,6 +98,7 @@ PORTED_MODULES = [
     "allmydata.storage.shares",
     "allmydata.test",
     "allmydata.test.cli",
+    "allmydata.test.cli.common",
     "allmydata.test.cli_node_api",
     "allmydata.test.common",
     "allmydata.test.common_util",

From cc176342d4eaf9d06082a8a95c62de2ce3cf4d7e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 11:22:12 -0400
Subject: [PATCH 79/81] Some progress towards test_system.py fully running on
 Python 3.

---
 src/allmydata/scripts/tahoe_unlink.py |  4 +--
 src/allmydata/test/common_util.py     |  2 +-
 src/allmydata/test/test_system.py     | 38 ++++++++++++---------------
 src/allmydata/util/_python3.py        |  5 ----
 4 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_unlink.py b/src/allmydata/scripts/tahoe_unlink.py
index bc1d43c9e..1ec92c69e 100644
--- a/src/allmydata/scripts/tahoe_unlink.py
+++ b/src/allmydata/scripts/tahoe_unlink.py
@@ -1,6 +1,6 @@
 from __future__ import print_function
 
-import urllib
+from urllib.parse import quote as url_quote
 from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
@@ -27,7 +27,7 @@ def unlink(options, command="unlink"):
 'tahoe %s' can only unlink directory entries, so a path must be given.""" % (command,), file=stderr)
         return 1
 
-    url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+    url = nodeurl + "uri/%s" % url_quote(rootcap)
     url += "/" + escape_path(path)
 
     resp = do_http("DELETE", url)
diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 95065710c..caafbb81d 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -85,7 +85,7 @@ def run_cli_native(verb, *args, **kwargs):
         bytes.
     """
     nodeargs = kwargs.pop("nodeargs", [])
-    encoding = kwargs.pop("encoding", "utf-8")
+    encoding = kwargs.pop("encoding", None) or "utf-8"
     return_bytes = kwargs.pop("return_bytes", False)
     verb = maybe_unicode_to_argv(verb)
     args = [maybe_unicode_to_argv(a) for a in args]
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 0ff1e06e9..8ffbf7063 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -1,12 +1,12 @@
 """
-Ported to Python 3, partially: test_filesystem* will be done in a future round.
+Ported to Python 3.
 """
 from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     # Don't import bytes since it causes issues on (so far unported) modules on Python 2.
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str  # noqa: F401
@@ -16,7 +16,6 @@ from six import ensure_text, ensure_str
 
 import os, re, sys, time, json
 from functools import partial
-from unittest import skipIf
 
 from bs4 import BeautifulSoup
 
@@ -1665,9 +1664,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(self.log, "did _check_publish_private")
         d.addCallback(self._test_web)
         d.addCallback(self._test_control)
-        if PY2:
-            # TODO when CLI is ported to Python 3, reenable.
-            d.addCallback(self._test_cli)
+        d.addCallback(self._test_cli)
         # P now has four top-level children:
         # P/personal/sekrit data
         # P/s2-ro/
@@ -2298,7 +2295,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_aliases_1(out_and_err):
             (out, err) = out_and_err
             self.failUnlessEqual(err, "")
-            self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % private_uri)
+            self.failUnlessEqual(out.strip(" \n"), "tahoe: %s" % str(private_uri, "ascii"))
         d.addCallback(_check_aliases_1)
 
         # now that that's out of the way, remove root_dir.cap and work with
@@ -2355,7 +2352,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             (out, err) = out_and_err
             self.failUnlessEqual(err, "")
             if filenum is not None:
-                self.failUnlessEqual(out, datas[filenum])
+                self.failUnlessEqual(out, str(datas[filenum], "ascii"))
             if data is not None:
                 self.failUnlessEqual(out, data)
 
@@ -2369,7 +2366,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             uri0 = out.strip()
             return run(None, "get", uri0)
         d.addCallback(_put_out)
-        d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], datas[0]))
+        d.addCallback(lambda out_err: self.failUnlessEqual(out_err[0], str(datas[0], "ascii")))
 
         d.addCallback(run, "put", files[1], "subdir/tahoe-file1")
         #  tahoe put bar tahoe:FOO
@@ -2411,14 +2408,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_outfile0(out_and_err):
             (out, err) = out_and_err
             data = open(outfile0,"rb").read()
-            self.failUnlessEqual(data, "data to be uploaded: file2\n")
+            self.failUnlessEqual(data, b"data to be uploaded: file2\n")
         d.addCallback(_check_outfile0)
         outfile1 = os.path.join(self.basedir, "outfile0")
         d.addCallback(run, "get", "tahoe:subdir/tahoe-file1", outfile1)
         def _check_outfile1(out_and_err):
             (out, err) = out_and_err
             data = open(outfile1,"rb").read()
-            self.failUnlessEqual(data, "data to be uploaded: file1\n")
+            self.failUnlessEqual(data, b"data to be uploaded: file1\n")
         d.addCallback(_check_outfile1)
 
         d.addCallback(run, "unlink", "tahoe-file0")
@@ -2455,7 +2452,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 if "file3" in l:
                     rw_uri = self._mutable_file3_uri
                     u = uri.from_string_mutable_filenode(rw_uri)
-                    ro_uri = u.get_readonly().to_string()
+                    ro_uri = str(u.get_readonly().to_string(), "ascii")
                     self.failUnless(ro_uri in l)
         d.addCallback(_check_ls_rouri)
 
@@ -2528,17 +2525,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         dn = os.path.join(self.basedir, "dir1")
         os.makedirs(dn)
         with open(os.path.join(dn, "rfile1"), "wb") as f:
-            f.write("rfile1")
+            f.write(b"rfile1")
         with open(os.path.join(dn, "rfile2"), "wb") as f:
-            f.write("rfile2")
+            f.write(b"rfile2")
         with open(os.path.join(dn, "rfile3"), "wb") as f:
-            f.write("rfile3")
+            f.write(b"rfile3")
         sdn2 = os.path.join(dn, "subdir2")
         os.makedirs(sdn2)
         with open(os.path.join(sdn2, "rfile4"), "wb") as f:
-            f.write("rfile4")
+            f.write(b"rfile4")
         with open(os.path.join(sdn2, "rfile5"), "wb") as f:
-            f.write("rfile5")
+            f.write(b"rfile5")
 
         # from disk into tahoe
         d.addCallback(run, "cp", "-r", dn, "tahoe:")
@@ -2615,7 +2612,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         return d
 
-    @skipIf(PY3, "Python 3 CLI support hasn't happened yet.")
     def test_filesystem_with_cli_in_subprocess(self):
         # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
 
@@ -2659,9 +2655,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_ls(res):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
-            self.failUnlessEqual(err, "", str(res))
-            self.failUnlessIn("tahoe-moved", out)
-            self.failIfIn("tahoe-file", out)
+            self.failUnlessEqual(err, b"", str(res))
+            self.failUnlessIn(b"tahoe-moved", out)
+            self.failIfIn(b"tahoe-file", out)
         d.addCallback(_check_ls)
         return d
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index d708a52ac..a18f50635 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -260,12 +260,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_storage",
     "allmydata.test.test_storage_client",
     "allmydata.test.test_storage_web",
-
-    # Only partially ported, test_filesystem_with_cli_in_subprocess isn't
-    # ported yet, nor is part of test_filesystem (the call to _test_cli). This
-    # should be done once CLI is ported.
     "allmydata.test.test_system",
-
     "allmydata.test.test_testing",
     "allmydata.test.test_time_format",
     "allmydata.test.test_tor_provider",

From 07b58e36192dfbc8cdf26ef8907b1c61e76eca3b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 11:34:58 -0400
Subject: [PATCH 80/81] All tests pass on Python 3.

---
 src/allmydata/scripts/tahoe_cp.py | 19 ++++++++++++++-----
 src/allmydata/test/test_system.py |  2 +-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_cp.py b/src/allmydata/scripts/tahoe_cp.py
index 0a6c67942..fb86ff0ec 100644
--- a/src/allmydata/scripts/tahoe_cp.py
+++ b/src/allmydata/scripts/tahoe_cp.py
@@ -5,7 +5,8 @@ from past.builtins import unicode
 import os.path
 from urllib.parse import quote as url_quote
 from collections import defaultdict
-from six.moves import cStringIO as StringIO
+from io import BytesIO
+
 from twisted.python.failure import Failure
 from allmydata.scripts.common import get_alias, escape_path, \
                                      DefaultAliasMarker, TahoeError
@@ -200,13 +201,21 @@ class TahoeFileSource(object):
 
     def open(self, caps_only):
         if caps_only:
-            return StringIO(self.readcap)
+            return BytesIO(self.readcap)
         url = self.nodeurl + "uri/" + url_quote(self.readcap)
         return GET_to_file(url)
 
     def bestcap(self):
         return self.writecap or self.readcap
 
+
+def seekable(file_like):
+    """Return whether the file-like object is seekable."""
+    return hasattr(file_like, "seek") and (
+        not hasattr(file_like, "seekable") or file_like.seekable()
+    )
+
+
 class TahoeFileTarget(object):
     def __init__(self, nodeurl, mutable, writecap, readcap, url):
         self.nodeurl = nodeurl
@@ -220,7 +229,7 @@ class TahoeFileTarget(object):
         assert self.url
         # our do_http() call currently requires a string or a filehandle with
         # a real .seek
-        if not hasattr(inf, "seek"):
+        if not seekable(inf):
             inf = inf.read()
         PUT(self.url, inf)
         # TODO: this always creates immutable files. We might want an option
@@ -306,7 +315,7 @@ class TahoeMissingTarget(object):
 
     def put_file(self, inf):
         # We want to replace this object in-place.
-        if not hasattr(inf, "seek"):
+        if not seekable(inf):
             inf = inf.read()
         PUT(self.url, inf)
         # TODO: this always creates immutable files. We might want an option
@@ -417,7 +426,7 @@ class TahoeDirectoryTarget(object):
     def put_file(self, name, inf):
         precondition(isinstance(name, unicode), name)
         url = self.nodeurl + "uri"
-        if not hasattr(inf, "seek"):
+        if not seekable(inf):
             inf = inf.read()
 
         if self.children is None:
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 8ffbf7063..12ae846eb 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -2579,7 +2579,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             (out, err) = out_and_err
             x = open(os.path.join(dn_copy2, "dir1", "subdir2", "rfile4")).read()
             y = uri.from_string_filenode(x)
-            self.failUnlessEqual(y.data, "rfile4")
+            self.failUnlessEqual(y.data, b"rfile4")
         d.addCallback(_check_capsonly)
 
         # and tahoe-to-tahoe

From 90240ae5efb7f6a0ecf4c17f193da95e56ec0c1d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 4 May 2021 12:03:12 -0400
Subject: [PATCH 81/81] Tests pass on Python 3.

---
 src/allmydata/scripts/slow_operation.py |  8 +++++++-
 src/allmydata/scripts/tahoe_manifest.py |  4 ++++
 src/allmydata/test/test_deepcheck.py    | 21 ++++++++++-----------
 src/allmydata/util/_python3.py          |  4 ----
 4 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/scripts/slow_operation.py b/src/allmydata/scripts/slow_operation.py
index 5d8c77538..b4b2f8196 100644
--- a/src/allmydata/scripts/slow_operation.py
+++ b/src/allmydata/scripts/slow_operation.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+from future.utils import PY3
 from past.builtins import unicode
 from six import ensure_str
 
@@ -78,8 +79,13 @@ class SlowOperationRunner(object):
         if not data["finished"]:
             return False
         if self.options.get("raw"):
+            if PY3:
+                # need to write bytes!
+                stdout = stdout.buffer
             if is_printable_ascii(jdata):
-                print(jdata, file=stdout)
+                stdout.write(jdata)
+                stdout.write(b"\n")
+                stdout.flush()
             else:
                 print("The JSON response contained unprintable characters:\n%s" % quote_output(jdata), file=stderr)
             return True
diff --git a/src/allmydata/scripts/tahoe_manifest.py b/src/allmydata/scripts/tahoe_manifest.py
index b837e648a..966583244 100644
--- a/src/allmydata/scripts/tahoe_manifest.py
+++ b/src/allmydata/scripts/tahoe_manifest.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+from future.utils import PY3
 from past.builtins import unicode
 
 from urllib.parse import quote as url_quote
@@ -51,6 +52,9 @@ class ManifestStreamer(LineOnlyReceiver, object):
         #print("RESP", dir(resp))
         # use Twisted to split this into lines
         self.in_error = False
+        # Writing bytes, so need binary stdout.
+        if PY3:
+            stdout = stdout.buffer
         while True:
             chunk = resp.read(100)
             if not chunk:
diff --git a/src/allmydata/test/test_deepcheck.py b/src/allmydata/test/test_deepcheck.py
index baee1acbe..652e51ea5 100644
--- a/src/allmydata/test/test_deepcheck.py
+++ b/src/allmydata/test/test_deepcheck.py
@@ -17,10 +17,10 @@ from __future__ import unicode_literals
 # (Pdb) pp data
 # '334:12:b\'mutable-good\',90:URI:SSK-RO:...
 from past.builtins import unicode as str
-from future.utils import PY3, PY2
+from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
-
+from six import ensure_text
 
 import os, json
 from urllib.parse import quote as url_quote
@@ -170,7 +170,8 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
         return data
 
     def parse_streamed_json(self, s):
-        for unit in s.split(b"\n"):
+        s = ensure_text(s)
+        for unit in s.split("\n"):
             if not unit:
                 # stream should end with a newline, so split returns ""
                 continue
@@ -746,8 +747,6 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
     def do_test_cli_good(self, ignored):
         d = defer.succeed(None)
-        if PY3:  # TODO fixme once Python 3 CLI porting is done
-            return d
         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
@@ -758,7 +757,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def _check_manifest_storage_index(self, out):
-        lines = [l for l in out.split(b"\n") if l]
+        lines = [l.encode("utf-8") for l in out.split("\n") if l]
         self.failUnlessEqual(len(lines), 3)
         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
@@ -769,7 +768,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split(b"\n") if l]
+            lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 8)
             caps = {}
             for l in lines:
@@ -778,7 +777,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
                 except ValueError:
                     cap = l.strip()
                     path = ""
-                caps[cap] = path
+                caps[cap.encode("ascii")] = path
             self.failUnless(self.root.get_uri() in caps)
             self.failUnlessEqual(caps[self.root.get_uri()], "")
             self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
@@ -814,7 +813,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split(b"\n") if l]
+            lines = [l.encode("utf-8") for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 3)
             self.failUnless(self.root.get_verify_cap().to_string() in lines)
             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
@@ -827,7 +826,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split(b"\n") if l]
+            lines = [l.encode("utf-8") for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 3)
             self.failUnless(self.root.get_repair_cap().to_string() in lines)
             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
@@ -839,7 +838,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         d = self.do_cli("stats", self.root_uri)
         def _check3(args):
             (rc, out, err) = args
-            lines = [l.strip() for l in out.split(b"\n") if l]
+            lines = [l.strip() for l in out.split("\n") if l]
             self.failUnless("count-immutable-files: 1" in lines)
             self.failUnless("count-mutable-files: 1" in lines)
             self.failUnless("count-literal-files: 3" in lines)
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index a18f50635..5f78af626 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -218,11 +218,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_consumer",
     "allmydata.test.test_crawler",
     "allmydata.test.test_crypto",
-
-    # Only partially ported, CLI-using test code is disabled for now until CLI
-    # is ported.
     "allmydata.test.test_deepcheck",
-
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",
     "allmydata.test.test_dirnode",