From 512897eca0310ce6ddfe262520f3c21bcf345ccf Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 19 Jan 2021 13:46:32 -0500
Subject: [PATCH 01/50] news fragment

---
 newsfragments/3592.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3592.minor

diff --git a/newsfragments/3592.minor b/newsfragments/3592.minor
new file mode 100644
index 000000000..e69de29bb

From 61d5f920bb9db703edb9f3aed73f2d0488ee0d2c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 19 Jan 2021 14:28:16 -0500
Subject: [PATCH 02/50] Add tests for the tag construction code and make it a
 bit safer

Check for sane inputs, reject insane ones
---
 src/allmydata/test/test_hashutil.py | 36 +++++++++++++++++++++++++++++
 src/allmydata/util/hashutil.py      | 29 ++++++++++++++++++++++-
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py
index 6ec861c9f..6ac73ae60 100644
--- a/src/allmydata/test/test_hashutil.py
+++ b/src/allmydata/test/test_hashutil.py
@@ -126,6 +126,42 @@ class HashUtilTests(unittest.TestCase):
             base32.a2b(b"2ckv3dfzh6rgjis6ogfqhyxnzy"),
         )
 
+    def test_convergence_hasher_tag(self):
+        """
+        ``_convergence_hasher_tag`` constructs the convergence hasher tag from a
+        unique prefix, the required, total, and segment size parameters, and a
+        convergence secret.
+        """
+        self.assertEqual(
+            "allmydata_immutable_content_to_key_with_added_secret_v1+"
+            "16:\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42,"
+            "9:3,10,1024,",
+            hashutil._convergence_hasher_tag(
+                k=3,
+                n=10,
+                segsize=1024,
+                convergence=b"\x42" * 16,
+            ),
+        )
+
+    def test_convergence_hasher_out_of_bounds(self):
+        """
+        ``_convergence_hasher_tag`` raises ``ValueError`` if k or n is not between
+        1 and 256 inclusive or if k is greater than n.
+        """
+        segsize = 1024
+        secret = b"\x42" * 16
+        for bad_k in (0, 2, 257):
+            with self.assertRaises(ValueError):
+                hashutil._convergence_hasher_tag(
+                    k=bad_k, n=1, segsize=segsize, convergence=secret,
+                )
+        for bad_n in (0, 1, 257):
+            with self.assertRaises(ValueError):
+                hashutil._convergence_hasher_tag(
+                    k=2, n=bad_n, segsize=segsize, convergence=secret,
+                )
+
     def test_known_answers(self):
         """
         Verify backwards compatibility by comparing hash outputs for some
diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py
index ebb2f12af..f82c04efd 100644
--- a/src/allmydata/util/hashutil.py
+++ b/src/allmydata/util/hashutil.py
@@ -176,10 +176,37 @@ def convergence_hash(k, n, segsize, data, convergence):
     return h.digest()
 
 
-def convergence_hasher(k, n, segsize, convergence):
+def _convergence_hasher_tag(k, n, segsize, convergence):
+    """
+    Create the convergence hashing tag.
+
+    :param int k: Required shares.
+    :param int n: Total shares.
+    :param int segsize: Maximum segment size.
+    :param bytes convergence: The convergence secret.
+
+    :return bytes: The bytestring to use as a tag in the convergence hash.
+    """
     assert isinstance(convergence, bytes)
+    if k > n:
+        raise ValueError(
+            "k > n not allowed; k = {}, n = {}".format(k, n),
+        )
+    if k < 1 or n < 1:
+        raise ValueError(
+            "k, n < 1 not allowed; k = {}, n = {}".format(k, n),
+        )
+    if k > 256 or n > 256:
+        raise ValueError(
+            "k, n > 256 not allowed; k = {}, n = {}".format(k, n),
+        )
     param_tag = netstring(b"%d,%d,%d" % (k, n, segsize))
     tag = CONVERGENT_ENCRYPTION_TAG + netstring(convergence) + param_tag
+    return tag
+
+
+def convergence_hasher(k, n, segsize, convergence):
+    tag = _convergence_hasher_tag(k, n, segsize, convergence)
     return tagged_hasher(tag, KEYLEN)
 
 

From 755de5edafe7589ecf2202a05d17a403913b31fd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 22 Jan 2021 09:58:51 -0500
Subject: [PATCH 03/50] Start of passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 32 ++++++++++++++++--------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 2f000b7a1..244478a55 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -275,13 +275,13 @@ class FakeClient(_Client):  # type: ignore  # tahoe-lafs/ticket/3573
             node_config=EMPTY_CLIENT_CONFIG,
         )
         # fake knowledge of another server
-        self.storage_broker.test_add_server("other_nodeid",
+        self.storage_broker.test_add_server(b"other_nodeid",
             FakeDisplayableServer(
                 serverid=b"other_nodeid", nickname=u"other_nickname \u263B", connected = True,
                 last_connect_time = 10, last_loss_time = 20, last_rx_time = 30))
-        self.storage_broker.test_add_server("disconnected_nodeid",
+        self.storage_broker.test_add_server(b"disconnected_nodeid",
             FakeDisplayableServer(
-                serverid="disconnected_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
+                serverid=b"disconnected_nodeid", nickname=u"disconnected_nickname \u263B", connected = False,
                 last_connect_time = None, last_loss_time = 25, last_rx_time = 35))
         self.introducer_client = None
         self.history = FakeHistory()
@@ -665,6 +665,8 @@ class MultiFormatResourceTests(TrialTestCase):
     Tests for ``MultiFormatResource``.
     """
     def render(self, resource, **queryargs):
+        # Query arguments in real twisted.web requests have byte keys.
+        queryargs = {k.encode("utf-8"): v for (k, v) in queryargs.items()}
         return self.successResultOf(render(resource, queryargs))
 
     def resource(self):
@@ -675,13 +677,13 @@ class MultiFormatResourceTests(TrialTestCase):
         class Content(MultiFormatResource):
 
             def render_HTML(self, req):
-                return "html"
+                return b"html"
 
             def render_A(self, req):
-                return "a"
+                return b"a"
 
             def render_B(self, req):
-                return "b"
+                return b"b"
 
         return Content()
 
@@ -693,7 +695,7 @@ class MultiFormatResourceTests(TrialTestCase):
         """
         resource = self.resource()
         resource.formatArgument = "foo"
-        self.assertEqual("a", self.render(resource, foo=["a"]))
+        self.assertEqual(b"a", self.render(resource, foo=[b"a"]))
 
 
     def test_default_format_argument(self):
@@ -702,7 +704,7 @@ class MultiFormatResourceTests(TrialTestCase):
         then the ``t`` argument is used.
         """
         resource = self.resource()
-        self.assertEqual("a", self.render(resource, t=["a"]))
+        self.assertEqual(b"a", self.render(resource, t=[b"a"]))
 
 
     def test_no_format(self):
@@ -711,7 +713,7 @@ class MultiFormatResourceTests(TrialTestCase):
         been defined, the base rendering behavior is used (``render_HTML``).
         """
         resource = self.resource()
-        self.assertEqual("html", self.render(resource))
+        self.assertEqual(b"html", self.render(resource))
 
 
     def test_default_format(self):
@@ -722,7 +724,7 @@ class MultiFormatResourceTests(TrialTestCase):
         """
         resource = self.resource()
         resource.formatDefault = "b"
-        self.assertEqual("b", self.render(resource))
+        self.assertEqual(b"b", self.render(resource))
 
 
     def test_explicit_none_format_renderer(self):
@@ -732,7 +734,7 @@ class MultiFormatResourceTests(TrialTestCase):
         """
         resource = self.resource()
         resource.render_FOO = None
-        self.assertEqual("html", self.render(resource, t=["foo"]))
+        self.assertEqual(b"html", self.render(resource, t=[b"foo"]))
 
 
     def test_unknown_format(self):
@@ -741,15 +743,15 @@ class MultiFormatResourceTests(TrialTestCase):
         returned.
         """
         resource = self.resource()
-        response_body = self.render(resource, t=["foo"])
+        response_body = self.render(resource, t=[b"foo"])
         self.assertIn(
-            "<title>400 - Bad Format</title>", response_body,
+            b"<title>400 - Bad Format</title>", response_body,
         )
         self.assertIn(
-            "Unknown t value:", response_body,
+            b"Unknown t value:", response_body,
         )
         self.assertIn(
-            "'foo'", response_body,
+            b"'foo'", response_body,
         )
 
 

From 011b027c392ea1b3c182e419b1db91922a4f9465 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 22 Jan 2021 10:14:28 -0500
Subject: [PATCH 04/50] More passing tests on Python 3.

---
 src/allmydata/test/common.py       | 14 ++---
 src/allmydata/test/web/test_web.py | 95 +++++++++++++++---------------
 2 files changed, 56 insertions(+), 53 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index fde92fb59..f7f339b98 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -566,12 +566,12 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
         self.file_types[self.storage_index] = version
         initial_contents = self._get_initial_contents(contents)
         data = initial_contents.read(initial_contents.get_size())
-        data = "".join(data)
+        data = b"".join(data)
         self.all_contents[self.storage_index] = data
         return defer.succeed(self)
     def _get_initial_contents(self, contents):
         if contents is None:
-            return MutableData("")
+            return MutableData(b"")
 
         if IMutableUploadable.providedBy(contents):
             return contents
@@ -625,7 +625,7 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
     def raise_error(self):
         pass
     def get_writekey(self):
-        return "\x00"*16
+        return b"\x00"*16
     def get_size(self):
         return len(self.all_contents[self.storage_index])
     def get_current_size(self):
@@ -644,7 +644,7 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
         return self.file_types[self.storage_index]
 
     def check(self, monitor, verify=False, add_lease=False):
-        s = StubServer("\x00"*20)
+        s = StubServer(b"\x00"*20)
         r = CheckResults(self.my_uri, self.storage_index,
                          healthy=True, recoverable=True,
                          count_happiness=10,
@@ -655,7 +655,7 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
                          count_recoverable_versions=1,
                          count_unrecoverable_versions=0,
                          servers_responding=[s],
-                         sharemap={"seq1-abcd-sh0": [s]},
+                         sharemap={b"seq1-abcd-sh0": [s]},
                          count_wrong_shares=0,
                          list_corrupt_shares=[],
                          count_corrupt_shares=0,
@@ -709,7 +709,7 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
     def overwrite(self, new_contents):
         assert not self.is_readonly()
         new_data = new_contents.read(new_contents.get_size())
-        new_data = "".join(new_data)
+        new_data = b"".join(new_data)
         self.all_contents[self.storage_index] = new_data
         return defer.succeed(None)
     def modify(self, modifier):
@@ -740,7 +740,7 @@ class FakeMutableFileNode(object):  # type: ignore # incomplete implementation
     def update(self, data, offset):
         assert not self.is_readonly()
         def modifier(old, servermap, first_time):
-            new = old[:offset] + "".join(data.read(data.get_size()))
+            new = old[:offset] + b"".join(data.read(data.get_size()))
             new += old[len(new):]
             return new
         return self.modify(modifier)
diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 244478a55..bcf4db1bf 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1,8 +1,11 @@
 from __future__ import print_function
 
-import os.path, re, urllib, time
+from past.builtins import unicode
+
+import os.path, re, time
 import json
 import treq
+from urllib.parse import quote as urlquote, unquote as urlunquote
 
 from bs4 import BeautifulSoup
 
@@ -115,8 +118,8 @@ class FakeUploader(service.Service):
                                       servermap={},
                                       timings={},
                                       uri_extension_data={},
-                                      uri_extension_hash="fake",
-                                      verifycapstr="fakevcap")
+                                      uri_extension_hash=b"fake",
+                                      verifycapstr=b"fakevcap")
             ur.set_uri(n.get_uri())
             return ur
         d.addCallback(_got_data)
@@ -297,12 +300,12 @@ class FakeClient(_Client):  # type: ignore  # tahoe-lafs/ticket/3573
         self.addService(FakeStorageServer(self.nodeid, self.nickname))
 
     def get_long_nodeid(self):
-        return "v0-nodeid"
+        return b"v0-nodeid"
     def get_long_tubid(self):
-        return "tubid"
+        return u"tubid"
 
     def get_auth_token(self):
-        return 'a fake debug auth token'
+        return b'a fake debug auth token'
 
     def startService(self):
         return service.MultiService.startService(self)
@@ -340,7 +343,7 @@ class WebMixin(TimezoneMixin):
         def _then(res):
             self.public_root = res[0][1]
             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
-            self.public_url = "/uri/" + self.public_root.get_uri()
+            self.public_url = "/uri/" + unicode(self.public_root.get_uri(), "ascii")
             self.private_root = res[1][1]
 
             foo = res[2][1]
@@ -365,7 +368,7 @@ class WebMixin(TimezoneMixin):
 
             # mdmf
             self.QUUX_CONTENTS, n, self._quux_txt_uri, self._quux_txt_readonly_uri = self.makefile_mutable(0, mdmf=True)
-            assert self._quux_txt_uri.startswith("URI:MDMF")
+            assert self._quux_txt_uri.startswith(b"URI:MDMF")
             foo.set_uri(u"quux.txt", self._quux_txt_uri, self._quux_txt_readonly_uri)
 
             foo.set_uri(u"empty", res[3][1].get_uri(),
@@ -382,7 +385,7 @@ class WebMixin(TimezoneMixin):
             # filenode to test for html encoding issues
             self._htmlname_unicode = u"<&weirdly'named\"file>>>_<iframe />.txt"
             self._htmlname_raw = self._htmlname_unicode.encode('utf-8')
-            self._htmlname_urlencoded = urllib.quote(self._htmlname_raw, '')
+            self._htmlname_urlencoded = urlquote(self._htmlname_raw, '')
             self.HTMLNAME_CONTENTS, n, self._htmlname_txt_uri = self.makefile(0)
             foo.set_uri(self._htmlname_unicode, self._htmlname_txt_uri, self._htmlname_txt_uri)
 
@@ -416,7 +419,7 @@ class WebMixin(TimezoneMixin):
             # public/foo/sub/baz.txt
             # public/reedownlee/
             # public/reedownlee/nor
-            self.NEWFILE_CONTENTS = "newfile contents\n"
+            self.NEWFILE_CONTENTS = b"newfile contents\n"
 
             return foo.get_metadata_for(u"bar.txt")
         d.addCallback(_then)
@@ -429,12 +432,12 @@ class WebMixin(TimezoneMixin):
         return self.s.all_contents
 
     def makefile(self, number):
-        contents = "contents of file %s\n" % number
+        contents = b"contents of file %d\n" % number
         n = create_chk_filenode(contents, self.get_all_contents())
         return contents, n, n.get_uri()
 
     def makefile_mutable(self, number, mdmf=False):
-        contents = "contents of mutable file %s\n" % number
+        contents = b"contents of mutable file %d\n" % number
         n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
         return contents, n, n.get_uri(), n.get_readonly_uri()
 
@@ -1384,8 +1387,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_named(self):
-        base = "/file/%s" % urllib.quote(self._bar_txt_uri)
-        base2 = "/named/%s" % urllib.quote(self._bar_txt_uri)
+        base = "/file/%s" % urlquote(self._bar_txt_uri)
+        base2 = "/named/%s" % urlquote(self._bar_txt_uri)
         d = self.GET(base + "/@@name=/blah.txt")
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(base + "/blah.txt"))
@@ -1398,14 +1401,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res: self.GET(save_url))
         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
         u_filename = u"n\u00e9wer.txt" # n e-acute w e r . t x t
-        u_fn_e = urllib.quote(u_filename.encode("utf-8"))
+        u_fn_e = urlquote(u_filename.encode("utf-8"))
         u_url = base + "?save=true&filename=" + u_fn_e
         d.addCallback(lambda res: self.GET(u_url))
         d.addCallback(self.failUnlessIsBarDotTxt) # TODO: check headers
         return d
 
     def test_PUT_FILEURL_named_bad(self):
-        base = "/file/%s" % urllib.quote(self._bar_txt_uri)
+        base = "/file/%s" % urlquote(self._bar_txt_uri)
         d = self.shouldFail2(error.Error, "test_PUT_FILEURL_named_bad",
                              "400 Bad Request",
                              "/file can only be used with GET or HEAD",
@@ -1414,7 +1417,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
 
     def test_GET_DIRURL_named_bad(self):
-        base = "/file/%s" % urllib.quote(self._foo_uri)
+        base = "/file/%s" % urlquote(self._foo_uri)
         d = self.shouldFail2(error.Error, "test_PUT_DIRURL_named_bad",
                              "400 Bad Request",
                              "is not a file-cap",
@@ -1431,7 +1434,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_GET_unhandled_URI_named(self):
         contents, n, newuri = self.makefile(12)
         verifier_cap = n.get_verify_cap().to_string()
-        base = "/file/%s" % urllib.quote(verifier_cap)
+        base = "/file/%s" % urlquote(verifier_cap)
         # client.create_node_from_uri() can't handle verify-caps
         d = self.shouldFail2(error.Error, "GET_unhandled_URI_named",
                              "400 Bad Request", "is not a file-cap",
@@ -1441,7 +1444,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_GET_unhandled_URI(self):
         contents, n, newuri = self.makefile(12)
         verifier_cap = n.get_verify_cap().to_string()
-        base = "/uri/%s" % urllib.quote(verifier_cap)
+        base = "/uri/%s" % urlquote(verifier_cap)
         # client.create_node_from_uri() can't handle verify-caps
         d = self.shouldFail2(error.Error, "test_GET_unhandled_URI",
                              "400 Bad Request",
@@ -1450,31 +1453,31 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILE_URI(self):
-        base = "/uri/%s" % urllib.quote(self._bar_txt_uri)
+        base = "/uri/%s" % urlquote(self._bar_txt_uri)
         d = self.GET(base)
         d.addCallback(self.failUnlessIsBarDotTxt)
         return d
 
     def test_GET_FILE_URI_mdmf(self):
-        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        base = "/uri/%s" % urlquote(self._quux_txt_uri)
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urllib.quote("%s:RANDOMSTUFF" % self._quux_txt_uri)
+        base = "/uri/%s" % urlquote("%s:RANDOMSTUFF" % self._quux_txt_uri)
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_mdmf_readonly(self):
-        base = "/uri/%s" % urllib.quote(self._quux_txt_readonly_uri)
+        base = "/uri/%s" % urlquote(self._quux_txt_readonly_uri)
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
+        base = "/uri/%s/boguschild" % urlquote(self._bar_txt_uri)
         errmsg = "Files have no children named 'boguschild'"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "400 Bad Request", errmsg,
@@ -1482,7 +1485,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urllib.quote(self._bar_txt_uri)
+        base = "/uri/%s/boguschild" % urlquote(self._bar_txt_uri)
         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "409 Conflict", errmsg,
@@ -1490,7 +1493,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf(self):
-        base = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        base = "/uri/%s" % urlquote(self._quux_txt_uri)
         self._quux_new_contents = "new_contents"
         d = self.GET(base)
         d.addCallback(lambda res:
@@ -1504,7 +1507,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urllib.quote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
+        base = "/uri/%s" % urlquote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
         self._quux_new_contents = "new_contents"
         d = self.GET(base)
         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
@@ -1555,7 +1558,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             return d
 
         def _get_etag(uri, t=''):
-            targetbase = "/uri/%s?t=%s" % (urllib.quote(uri.strip()), t)
+            targetbase = "/uri/%s?t=%s" % (urlquote(uri.strip()), t)
             d = self.GET(targetbase, return_response=True, followRedirect=True)
             def _just_the_etag(result):
                 data, response, headers = result
@@ -1900,7 +1903,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_json_mdmf(self):
-        d = self.GET("/uri/%s?t=json" % urllib.quote(self._quux_txt_uri))
+        d = self.GET("/uri/%s?t=json" % urlquote(self._quux_txt_uri))
         d.addCallback(self.failUnlessIsQuuxJSON)
         return d
 
@@ -2011,7 +2014,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         )
 
         # the FILE reference points to a URI, but it should end in bar.txt
-        bar_url = "{}/file/{}/@@named=/bar.txt".format(root, urllib.quote(self._bar_txt_uri))
+        bar_url = "{}/file/{}/@@named=/bar.txt".format(root, urlquote(self._bar_txt_uri))
         self.assertTrue(
             any(
                 a.text == u"bar.txt"
@@ -2024,7 +2027,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 for td in soup.find_all(u"td", {u"align": u"right"})
             )
         )
-        foo_url = urllib.quote("{}/uri/{}/".format(root, self._foo_uri))
+        foo_url = urlquote("{}/uri/{}/".format(root, self._foo_uri))
         forms = soup.find_all(u"form", {u"action": foo_url})
         found = []
         for form in forms:
@@ -2038,7 +2041,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             {u"unlink", u"rename/relink"}
         )
 
-        sub_url = "{}/uri/{}/".format(root, urllib.quote(self._sub_uri))
+        sub_url = "{}/uri/{}/".format(root, urlquote(self._sub_uri))
         self.assertTrue(
             any(
                 td.findNextSibling()(u"a")[0][u"href"] == sub_url
@@ -2775,11 +2778,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check)
         def _check2(data):
             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
-            return self.GET("/uri/%s" % urllib.quote(self.filecap))
+            return self.GET("/uri/%s" % urlquote(self.filecap))
         d.addCallback(_check2)
         def _check3(data):
             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
-            return self.GET("/file/%s" % urllib.quote(self.filecap))
+            return self.GET("/file/%s" % urlquote(self.filecap))
         d.addCallback(_check3)
         def _check4(data):
             self.failUnlessReallyEqual(data, self.NEWFILE_CONTENTS)
@@ -2980,7 +2983,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         # make sure we can get to it from /uri/URI
         d.addCallback(lambda res:
-                      self.GET("/uri/%s" % urllib.quote(self._mutable_uri)))
+                      self.GET("/uri/%s" % urlquote(self._mutable_uri)))
         d.addCallback(lambda res:
                       self.failUnlessReallyEqual(res, NEW2_CONTENTS))
 
@@ -3190,24 +3193,24 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         self.failUnlessIn(redir_url, res)
 
     def test_POST_FILEURL_mdmf_check(self):
-        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        quux_url = "/uri/%s" % urlquote(self._quux_txt_uri)
         d = self.POST(quux_url, t="check")
         def _check(res):
             self.failUnlessIn("Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % self._quux_txt_uri)
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check"))
         d.addCallback(_check)
         return d
 
     def test_POST_FILEURL_mdmf_check_and_repair(self):
-        quux_url = "/uri/%s" % urllib.quote(self._quux_txt_uri)
+        quux_url = "/uri/%s" % urlquote(self._quux_txt_uri)
         d = self.POST(quux_url, t="check", repair="true")
         def _check(res):
             self.failUnlessIn("Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urllib.quote("%s:3:131073" % self._quux_txt_uri)
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % self._quux_txt_uri)
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check", repair="true"))
         d.addCallback(_check)
@@ -3506,7 +3509,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         url = self.webish_url + "/uri?t=mkdir&redirect_to_result=true"
         target = yield self.shouldRedirectTo(url, None, method="post",
                                              code=http.SEE_OTHER)
-        target = urllib.unquote(target)
+        target = urlunquote(target)
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     @inlineCallbacks
@@ -3516,7 +3519,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                              method="post",
                                              data=body, headers=headers,
                                              code=http.SEE_OTHER)
-        target = urllib.unquote(target)
+        target = urlunquote(target)
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def _make_readonly(self, u):
@@ -3697,7 +3700,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         target = yield self.shouldRedirectTo(url, None,
                                              method="post",
                                              code=http.SEE_OTHER)
-        target = urllib.unquote(target)
+        target = urlunquote(target)
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def test_POST_mkdir_replace(self): # return value?
@@ -4270,7 +4273,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         relbase = "/uri?uri=%s" % self._bar_txt_uri
         base = self.webish_url + relbase
         # this is supposed to give us a redirect to /uri/$URI, plus arguments
-        targetbase = self.webish_url + "/uri/%s" % urllib.quote(self._bar_txt_uri)
+        targetbase = self.webish_url + "/uri/%s" % urlquote(self._bar_txt_uri)
         yield self.shouldRedirectTo(base, targetbase)
         yield self.shouldRedirectTo(base+"&filename=bar.txt",
                                     targetbase+"?filename=bar.txt")
@@ -4524,7 +4527,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check1)
         def _check2(data):
             self.failUnlessReallyEqual(data, file_contents)
-            return self.GET("/uri/%s" % urllib.quote(self.filecap))
+            return self.GET("/uri/%s" % urlquote(self.filecap))
         d.addCallback(_check2)
         def _check3(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4772,7 +4775,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         response = yield req("POST", "/uri?format=sdmf&t=mkdir")
         dircap = yield response.content()
         assert dircap.startswith('URI:DIR2:')
-        dircap_uri = "/uri/?uri={}&t=json".format(urllib.quote(dircap))
+        dircap_uri = "/uri/?uri={}&t=json".format(urlquote(dircap))
 
         response = yield req(
             "GET",
@@ -4780,7 +4783,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         )
         self.assertEqual(
             response.request.absoluteURI,
-            self.webish_url + "/uri/{}?t=json".format(urllib.quote(dircap)))
+            self.webish_url + "/uri/{}?t=json".format(urlquote(dircap)))
         if response.code >= 400:
             raise Error(response.code, response=response.content())
 

From 27a5211385be31e80f77ffcd0dc6f35b0c1738f3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 22 Jan 2021 11:37:37 -0500
Subject: [PATCH 05/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index bcf4db1bf..c685992e0 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -649,7 +649,7 @@ class WebMixin(TimezoneMixin):
     def should404(self, res, which):
         if isinstance(res, failure.Failure):
             res.trap(error.Error)
-            self.failUnlessReallyEqual(res.value.status, "404")
+            self.failUnlessReallyEqual(res.value.status, b"404")
         else:
             self.fail("%s was supposed to Error(404), not get '%s'" %
                       (which, res))
@@ -657,7 +657,7 @@ class WebMixin(TimezoneMixin):
     def should302(self, res, which):
         if isinstance(res, failure.Failure):
             res.trap(error.Error)
-            self.failUnlessReallyEqual(res.value.status, "302")
+            self.failUnlessReallyEqual(res.value.status, b"302")
         else:
             self.fail("%s was supposed to Error(302), not get '%s'" %
                       (which, res))
@@ -1933,7 +1933,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_CSS_FILE(self):
         d = self.GET("/tahoe.css", followRedirect=True)
         def _check(res):
-            CSS_STYLE=re.compile('toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
+            CSS_STYLE=re.compile(b'toolbar\s{.+text-align:\scenter.+toolbar-item.+display:\sinline',re.DOTALL)
             self.failUnless(CSS_STYLE.search(res), res)
         d.addCallback(_check)
         return d
@@ -1976,7 +1976,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         self._check_upload_and_mkdir_forms(soup)
         toolbars = soup.find_all(u"li", {u"class": u"toolbar-item"})
         self.assertTrue(any(li.text == u"Return to Welcome page" for li in toolbars))
-        self.failUnlessIn("quux", data)
+        self.failUnlessIn(b"quux", data)
 
     @inlineCallbacks
     def test_GET_DIRECTORY_html_filenode_encoding(self):
@@ -4774,7 +4774,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         response = yield req("POST", "/uri?format=sdmf&t=mkdir")
         dircap = yield response.content()
-        assert dircap.startswith('URI:DIR2:')
+        assert dircap.startswith(b'URI:DIR2:')
         dircap_uri = "/uri/?uri={}&t=json".format(urlquote(dircap))
 
         response = yield req(

From 4e15b3706279f9751bfbb42579ff8133f03f342b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 22 Jan 2021 11:59:26 -0500
Subject: [PATCH 06/50] Bit more progress on Python 3.

---
 src/allmydata/test/web/test_web.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index c685992e0..226169f85 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -503,7 +503,7 @@ class WebMixin(TimezoneMixin):
                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
         kids = dict( [(unicode(name),value)
                       for (name,value)
-                      in data[1]["children"].iteritems()] )
+                      in data[1]["children"].items()] )
         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
         self.failUnlessIn("metadata", kids[u"sub"][1])
         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
@@ -576,7 +576,7 @@ class WebMixin(TimezoneMixin):
         form.append('')
         form.append('UTF-8')
         form.append(sep)
-        for name, value in fields.iteritems():
+        for name, value in fields.items():
             if isinstance(value, tuple):
                 filename, value = value
                 form.append('Content-Disposition: form-data; name="%s"; '
@@ -2027,7 +2027,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 for td in soup.find_all(u"td", {u"align": u"right"})
             )
         )
-        foo_url = urlquote("{}/uri/{}/".format(root, self._foo_uri))
+        foo_url = urlquote("{}/uri/{}/".format(root, unicode(self._foo_uri, "ascii")))
         forms = soup.find_all(u"form", {u"action": foo_url})
         found = []
         for form in forms:
@@ -2076,7 +2076,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # look at an empty directory
         data = yield self.GET(self.public_url + "/foo/empty")
         soup = BeautifulSoup(data, 'html5lib')
-        self.failUnlessIn("directory is empty", data)
+        self.failUnlessIn(b"directory is empty", data)
         mkdir_inputs = soup.find_all(u"input", {u"type": u"hidden", u"name": u"t", u"value": u"mkdir"})
         self.assertEqual(1, len(mkdir_inputs))
         self.assertEqual(
@@ -2103,7 +2103,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     @inlineCallbacks
     def test_GET_DIRURL_badtype(self):
         url = self.webish_url + self.public_url + "/foo?t=bogus"
-        yield self.assertHTTPError(url, 400, "bad t=bogus")
+        yield self.assertHTTPError(url, 400, b"bad t=bogus")
 
     def test_GET_DIRURL_json(self):
         d = self.GET(self.public_url + "/foo?t=json")
@@ -2251,7 +2251,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                         "largest-immutable-file": 19,
                         "api-version": 1,
                         }
-            for k,v in expected.iteritems():
+            for k,v in expected.items():
                 self.failUnlessReallyEqual(stats[k], v,
                                            "stats[%s] was %s, not %s" %
                                            (k, stats[k], v))
@@ -2948,7 +2948,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessEqual(parsed[0], "dirnode")
             children = dict( [(unicode(name),value)
                               for (name,value)
-                              in parsed[1]["children"].iteritems()] )
+                              in parsed[1]["children"].items()] )
             self.failUnlessIn(u"new.txt", children)
             new_json = children[u"new.txt"]
             self.failUnlessEqual(new_json[0], "filenode")

From 4c5ea34af657a3131676f6eab7b3efc6f7e23f75 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 22 Jan 2021 15:07:03 -0500
Subject: [PATCH 07/50] More passing tests on Python 3.

---
 .../provisioning/provisioning.py                 |  2 +-
 src/allmydata/test/common.py                     |  2 ++
 src/allmydata/test/web/test_web.py               |  8 ++++----
 src/allmydata/web/common.py                      |  4 ++--
 src/allmydata/web/directory.py                   | 16 ++++++++--------
 src/allmydata/web/filenode.py                    |  4 ++--
 src/allmydata/web/operations.py                  |  2 +-
 src/allmydata/web/root.py                        |  4 ++--
 8 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/misc/operations_helpers/provisioning/provisioning.py b/misc/operations_helpers/provisioning/provisioning.py
index 37acd16d2..d6dfc4cd7 100644
--- a/misc/operations_helpers/provisioning/provisioning.py
+++ b/misc/operations_helpers/provisioning/provisioning.py
@@ -46,7 +46,7 @@ class ProvisioningTool(rend.Page):
         req = inevow.IRequest(ctx)
 
         def getarg(name, astype=int):
-            if req.method != "POST":
+            if req.method != b"POST":
                 return None
             if name in req.fields:
                 return astype(req.fields[name].value)
diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index f7f339b98..c28db9191 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -859,6 +859,8 @@ class WebErrorMixin(object):
         body = yield response.content()
         self.assertEquals(response.code, code)
         if response_substring is not None:
+            if isinstance(response_substring, unicode):
+                response_substring = response_substring.encode("utf-8")
             self.assertIn(response_substring, body)
         returnValue(body)
 
diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 226169f85..0437cdfe8 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -636,7 +636,7 @@ class WebMixin(TimezoneMixin):
                                        getattr(res.value, "response", ""),
                                        which))
                 if response_substring:
-                    self.failUnlessIn(response_substring, res.value.response,
+                    self.failUnlessIn(response_substring, unicode(res.value.response, "utf-8"),
                                       "'%s' not in '%s' for test '%s'" % \
                                       (response_substring, res.value.response,
                                        which))
@@ -2054,8 +2054,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_GET_DIRURL_readonly(self):
         # look at a readonly directory
         data = yield self.GET(self.public_url + "/reedownlee", followRedirect=True)
-        self.failUnlessIn("(read-only)", data)
-        self.failIfIn("Upload a file", data)
+        self.failUnlessIn(b"(read-only)", data)
+        self.failIfIn(b"Upload a file", data)
 
     @inlineCallbacks
     def test_GET_DIRURL_readonly_dir(self):
@@ -2090,7 +2090,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
         data = yield self.GET("/uri/" + tiny_litdir_uri, followRedirect=True)
         soup = BeautifulSoup(data, 'html5lib')
-        self.failUnlessIn('(immutable)', data)
+        self.failUnlessIn(b'(immutable)', data)
         file_links = list(
             td.findNextSibling()(u"a")[0]
             for td in soup.find_all(u"td")
diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index 57118d1d4..4a06d4ffd 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -283,8 +283,8 @@ def render_time_attr(t):
 # actual exception). The latter is growing increasingly annoying.
 
 def should_create_intermediate_directories(req):
-    t = get_arg(req, "t", "").strip()
-    return bool(req.method in ("PUT", "POST") and
+    t = unicode(get_arg(req, "t", "").strip(), "ascii")
+    return bool(req.method in (b"PUT", b"POST") and
                 t not in ("delete", "rename", "rename-form", "check"))
 
 def humanize_exception(exc):
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 981c8ef56..2a24cfa3b 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -135,7 +135,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name
         nonterminal = not terminal  #len(req.postpath) > 0
 
-        t = get_arg(req, b"t", b"").strip()
+        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
         if isinstance(node_or_failure, Failure):
             f = node_or_failure
             f.trap(NoSuchChildError)
@@ -150,10 +150,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             else:
                 # terminal node
                 terminal_requests = (
-                    ("POST", "mkdir"),
-                    ("PUT", "mkdir"),
-                    ("POST", "mkdir-with-children"),
-                    ("POST", "mkdir-immutable")
+                    (b"POST", "mkdir"),
+                    (b"PUT", "mkdir"),
+                    (b"POST", "mkdir-with-children"),
+                    (b"POST", "mkdir-immutable")
                 )
                 if (req.method, t) in terminal_requests:
                     # final directory
@@ -182,8 +182,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
                     )
                     return d
                 leaf_requests = (
-                    ("PUT",""),
-                    ("PUT","uri"),
+                    (b"PUT",""),
+                    (b"PUT","uri"),
                 )
                 if (req.method, t) in leaf_requests:
                     # we were trying to find the leaf filenode (to put a new
@@ -255,7 +255,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
 
     @render_exception
     def render_PUT(self, req):
-        t = get_arg(req, b"t", b"").strip()
+        t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
         if t == "mkdir":
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index 5bd575631..6111015ed 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -475,8 +475,8 @@ class FileDownloader(Resource, object):
                     size = contentsize
 
         req.setHeader("content-length", b"%d" % contentsize)
-        if req.method == "HEAD":
-            return ""
+        if req.method == b"HEAD":
+            return b""
 
         d = self.filenode.read(req, first, size)
 
diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py
index 2ba87c5ec..ca6826169 100644
--- a/src/allmydata/web/operations.py
+++ b/src/allmydata/web/operations.py
@@ -106,7 +106,7 @@ class OphandleTable(resource.Resource, service.Service):
         (monitor, renderer, when_added) = self.handles[ophandle]
 
         t = get_arg(req, "t", "status")
-        if t == "cancel" and req.method == "POST":
+        if t == "cancel" and req.method == b"POST":
             monitor.cancel()
             # return the status anyways, but release the handle
             self._release_ophandle(ophandle)
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index 9fb3ac9d3..c6f272381 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -177,7 +177,7 @@ class FileHandler(resource.Resource, object):
 
     @exception_to_child
     def getChild(self, name, req):
-        if req.method not in ("GET", "HEAD"):
+        if req.method not in (b"GET", b"HEAD"):
             raise WebError("/file can only be used with GET or HEAD")
         # 'name' must be a file URI
         try:
@@ -200,7 +200,7 @@ class IncidentReporter(MultiFormatResource):
 
     @render_exception
     def render(self, req):
-        if req.method != "POST":
+        if req.method != b"POST":
             raise WebError("/report_incident can only be used with POST")
 
         log.msg(format="User reports incident through web page: %(details)s",

From 3eb62f1d0f3809881bf7b0ad693ca4fc6ebd30bd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 25 Jan 2021 09:25:20 -0500
Subject: [PATCH 08/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 16 ++++++++--------
 src/allmydata/web/info.py          | 11 ++++++-----
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 0437cdfe8..4908ab0d7 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1644,35 +1644,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_info_mdmf(self):
-        d = self.GET("/uri/%s?t=info" % self._quux_txt_uri)
+        d = self.GET("/uri/%s?t=info" % unicode(self._quux_txt_uri, "ascii"))
         def _got(res):
-            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failUnlessIn(self._quux_txt_uri, res)
             self.failUnlessIn(self._quux_txt_readonly_uri, res)
         d.addCallback(_got)
         return d
 
     def test_GET_FILEURL_info_mdmf_readonly(self):
-        d = self.GET("/uri/%s?t=info" % self._quux_txt_readonly_uri)
+        d = self.GET("/uri/%s?t=info" % unicode(self._quux_txt_readonly_uri, "ascii"))
         def _got(res):
-            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failIfIn(self._quux_txt_uri, res)
             self.failUnlessIn(self._quux_txt_readonly_uri, res)
         d.addCallback(_got)
         return d
 
     def test_GET_FILEURL_info_sdmf(self):
-        d = self.GET("/uri/%s?t=info" % self._baz_txt_uri)
+        d = self.GET("/uri/%s?t=info" % unicode(self._baz_txt_uri, "ascii"))
         def _got(res):
-            self.failUnlessIn("mutable file (sdmf)", res)
+            self.failUnlessIn(b"mutable file (sdmf)", res)
             self.failUnlessIn(self._baz_txt_uri, res)
         d.addCallback(_got)
         return d
 
     def test_GET_FILEURL_info_mdmf_extensions(self):
-        d = self.GET("/uri/%s:STUFF?t=info" % self._quux_txt_uri)
+        d = self.GET("/uri/%s:STUFF?t=info" % unicode(self._quux_txt_uri, "ascii"))
         def _got(res):
-            self.failUnlessIn("mutable file (mdmf)", res)
+            self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failUnlessIn(self._quux_txt_uri, res)
             self.failUnlessIn(self._quux_txt_readonly_uri, res)
         d.addCallback(_got)
diff --git a/src/allmydata/web/info.py b/src/allmydata/web/info.py
index 7eccfea9e..e6bca0e04 100644
--- a/src/allmydata/web/info.py
+++ b/src/allmydata/web/info.py
@@ -1,5 +1,6 @@
 
-import os, urllib
+import os
+from urllib.parse import quote as urlquote
 
 from twisted.python.filepath import FilePath
 from twisted.web.template import tags as T, Element, renderElement, XMLFile, renderer
@@ -180,7 +181,7 @@ class MoreInfoElement(Element):
         else:
             return ""
         root = self.get_root(req)
-        quoted_uri = urllib.quote(node.get_uri())
+        quoted_uri = urlquote(node.get_uri())
         text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
         return T.li("Raw data as ", T.a("text/plain", href=text_plain_url))
 
@@ -196,7 +197,7 @@ class MoreInfoElement(Element):
     @renderer
     def check_form(self, req, tag):
         node = self.original
-        quoted_uri = urllib.quote(node.get_uri())
+        quoted_uri = urlquote(node.get_uri())
         target = self.get_root(req) + "/uri/" + quoted_uri
         if IDirectoryNode.providedBy(node):
             target += "/"
@@ -236,8 +237,8 @@ class MoreInfoElement(Element):
     def overwrite_form(self, req, tag):
         node = self.original
         root = self.get_root(req)
-        action = "%s/uri/%s" % (root, urllib.quote(node.get_uri()))
-        done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri()))
+        action = "%s/uri/%s" % (root, urlquote(node.get_uri()))
+        done_url = "%s/uri/%s?t=info" % (root, urlquote(node.get_uri()))
         overwrite = T.form(action=action, method="post",
                            enctype="multipart/form-data")(
             T.fieldset(

From b6788c4439e9506002b0dbc55d4b5221fc827e64 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 25 Jan 2021 09:55:55 -0500
Subject: [PATCH 09/50] More Python 3 fixes.

---
 src/allmydata/test/web/test_web.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 4908ab0d7..f4fd091c5 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -481,9 +481,9 @@ class WebMixin(TimezoneMixin):
             self.failIfIn("rw_uri", metadata)
         else:
             self.failUnlessIn("rw_uri", metadata)
-            self.failUnlessEqual(metadata['rw_uri'], self._quux_txt_uri)
+            self.failUnlessEqual(metadata['rw_uri'], unicode(self._quux_txt_uri, "ascii"))
         self.failUnlessIn("ro_uri", metadata)
-        self.failUnlessEqual(metadata['ro_uri'], self._quux_txt_readonly_uri)
+        self.failUnlessEqual(metadata['ro_uri'], unicode(self._quux_txt_readonly_uri, "ascii"))
         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 
     def failUnlessIsFooJSON(self, res):

From 59847edf9c52f30e5ce0c39af25b606e9daaafa3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 25 Jan 2021 10:06:05 -0500
Subject: [PATCH 10/50] More Python 3 fixes.

---
 src/allmydata/test/web/test_web.py | 6 +++---
 src/allmydata/web/root.py          | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index f4fd091c5..c8749c0ba 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1883,7 +1883,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # file is SDMF or MDMF
         d = self.PUT("/uri?format=mdmf",
                      self.NEWFILE_CONTENTS * 300000)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "ascii")))
         def _got_json(raw, version):
             data = json.loads(raw)
             assert "filenode" == data[0]
@@ -1898,12 +1898,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ignored:
             self.PUT("/uri?format=sdmf",
                       self.NEWFILE_CONTENTS * 300000))
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "ascii")))
         d.addCallback(_got_json, "SDMF")
         return d
 
     def test_GET_FILEURL_json_mdmf(self):
-        d = self.GET("/uri/%s?t=json" % urlquote(self._quux_txt_uri))
+        d = self.GET("/uri/%s?t=json" % urlquote(unicode(self._quux_txt_uri, "ascii")))
         d.addCallback(self.failUnlessIsQuuxJSON)
         return d
 
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index c6f272381..c0e614413 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -1,4 +1,5 @@
 from future.utils import PY3
+from past.builtins import unicode
 
 import os
 import time
@@ -97,7 +98,7 @@ class URIHandler(resource.Resource, object):
         either "PUT /uri" to create an unlinked file, or
         "PUT /uri?t=mkdir" to create an unlinked directory
         """
-        t = get_arg(req, "t", "").strip()
+        t = unicode(get_arg(req, "t", "").strip(), "utf-8")
         if t == "":
             file_format = get_format(req, "CHK")
             mutable_type = get_mutable_type(file_format)

From 58745fd373906b2699defe4f02ca7471a88a7f0d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 25 Jan 2021 10:32:28 -0500
Subject: [PATCH 11/50] More passing Python 3 tests.

---
 src/allmydata/test/web/test_web.py | 35 +++++++++++++++---------------
 src/allmydata/web/filenode.py      |  2 +-
 src/allmydata/web/root.py          |  2 +-
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index c8749c0ba..5123d3efe 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -3,7 +3,6 @@ from __future__ import print_function
 from past.builtins import unicode
 
 import os.path, re, time
-import json
 import treq
 from urllib.parse import quote as urlquote, unquote as urlunquote
 
@@ -26,7 +25,7 @@ from allmydata.immutable.downloader.status import DownloadStatus
 from allmydata.dirnode import DirectoryNode
 from allmydata.nodemaker import NodeMaker
 from allmydata.web.common import MultiFormatResource
-from allmydata.util import fileutil, base32, hashutil
+from allmydata.util import fileutil, base32, hashutil, jsonbytes as json
 from allmydata.util.consumer import download_to_data
 from allmydata.util.encodingutil import to_bytes
 from ...util.connection_status import ConnectionStatus
@@ -1465,19 +1464,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urlquote("%s:RANDOMSTUFF" % self._quux_txt_uri)
+        base = "/uri/%s" % urlquote("%s:RANDOMSTUFF" % unicode(self._quux_txt_uri, "ascii"))
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_mdmf_readonly(self):
-        base = "/uri/%s" % urlquote(self._quux_txt_readonly_uri)
+        base = "/uri/%s" % urlquote(unicode(self._quux_txt_readonly_uri, "ascii"))
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urlquote(self._bar_txt_uri)
+        base = "/uri/%s/boguschild" % urlquote(unicode(self._bar_txt_uri, "ascii"))
         errmsg = "Files have no children named 'boguschild'"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "400 Bad Request", errmsg,
@@ -1485,7 +1484,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urlquote(self._bar_txt_uri)
+        base = "/uri/%s/boguschild" % urlquote(unicode(self._bar_txt_uri, "ascii"))
         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "409 Conflict", errmsg,
@@ -1493,7 +1492,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf(self):
-        base = "/uri/%s" % urlquote(self._quux_txt_uri)
+        base = "/uri/%s" % urlquote(unicode(self._quux_txt_uri, "ascii"))
         self._quux_new_contents = "new_contents"
         d = self.GET(base)
         d.addCallback(lambda res:
@@ -1507,8 +1506,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urlquote("%s:EXTENSIONSTUFF" % self._quux_txt_uri)
-        self._quux_new_contents = "new_contents"
+        base = "/uri/%s" % urlquote("%s:EXTENSIONSTUFF" % unicode(self._quux_txt_uri, "ascii"))
+        self._quux_new_contents = b"new_contents"
         d = self.GET(base)
         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
         d.addCallback(lambda ignored: self.PUT(base, self._quux_new_contents))
@@ -1519,7 +1518,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_FILE_URI_mdmf_readonly(self):
         # We're not allowed to PUT things to a readonly cap.
-        base = "/uri/%s" % self._quux_txt_readonly_uri
+        base = "/uri/%s" % unicode(self._quux_txt_readonly_uri, "ascii")
         d = self.GET(base)
         d.addCallback(lambda res:
             self.failUnlessIsQuuxDotTxt(res))
@@ -1532,7 +1531,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_FILE_URI_sdmf_readonly(self):
         # We're not allowed to put things to a readonly cap.
-        base = "/uri/%s" % self._baz_txt_readonly_uri
+        base = "/uri/%s" % unicode(self._baz_txt_readonly_uri, "ascii")
         d = self.GET(base)
         d.addCallback(lambda res:
             self.failUnlessIsBazDotTxt(res))
@@ -1583,7 +1582,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         # use the ETag on GET
         def _check_match(ign):
-            uri = "/uri/%s" % self._bar_txt_uri
+            uri = "/uri/%s" % unicode(self._bar_txt_uri, "ascii")
             d = self.GET(uri, return_response=True)
             # extract the ETag
             d.addCallback(lambda data_code_headers:
@@ -4265,12 +4264,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         self.assertIn(response.code, codes)
         location = response.headers.getRawHeaders(b"location")[0]
         if target_location is not None:
-            self.assertEquals(location, target_location)
+            self.assertEquals(unicode(location, "ascii"), target_location)
         returnValue(location)
 
     @inlineCallbacks
     def test_GET_URI_form(self):
-        relbase = "/uri?uri=%s" % self._bar_txt_uri
+        relbase = "/uri?uri=%s" % unicode(self._bar_txt_uri, "ascii")
         base = self.webish_url + relbase
         # this is supposed to give us a redirect to /uri/$URI, plus arguments
         targetbase = self.webish_url + "/uri/%s" % urlquote(self._bar_txt_uri)
@@ -4284,7 +4283,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         data = yield self.GET(relbase, followRedirect=True)
         self.failUnlessIsBarDotTxt(data)
         self.log(None, "got file by uri, about to get dir by uri")
-        data = yield self.GET("/uri?uri=%s&t=json" % self._foo_uri,
+        data = yield self.GET("/uri?uri=%s&t=json" % unicode(self._foo_uri, "ascii"),
                               followRedirect=True)
         self.failUnlessIsFooJSON(data)
         self.log(None, "got dir by uri")
@@ -4318,7 +4317,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return res
 
     def test_GET_URI_URL(self):
-        base = "/uri/%s" % self._bar_txt_uri
+        base = "/uri/%s" % unicode(self._bar_txt_uri, "ascii")
         d = self.GET(base)
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
@@ -4328,14 +4327,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_URI_URL_dir(self):
-        base = "/uri/%s?t=json" % self._foo_uri
+        base = "/uri/%s?t=json" % unicode(self._foo_uri, "ascii")
         d = self.GET(base)
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 
     @inlineCallbacks
     def test_GET_URI_URL_missing(self):
-        base = "/uri/%s" % self._bad_file_uri
+        base = "/uri/%s" % unicode(self._bad_file_uri, "ascii")
         url = self.webish_url + base
         yield self.assertHTTPError(url, http.GONE, "NotEnoughSharesError")
         # TODO: how can we exercise both sides of WebDownloadTarget.fail
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index 6111015ed..b34048623 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -439,7 +439,7 @@ class FileDownloader(Resource, object):
             # bytes we were given in the URL. See the comment in
             # FileNodeHandler.render_GET for the sad details.
             req.setHeader("content-disposition",
-                          'attachment; filename="%s"' % self.filename)
+                          b'attachment; filename="%s"' % self.filename)
 
         filesize = self.filenode.get_size()
         assert isinstance(filesize, (int,long)), filesize
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index c0e614413..c5474a6cb 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -121,7 +121,7 @@ class URIHandler(resource.Resource, object):
         unlinked file or "POST /uri?t=mkdir" to create a
         new directory
         """
-        t = get_arg(req, "t", "").strip()
+        t = unicode(get_arg(req, "t", "").strip(), "ascii")
         if t in ("", "upload"):
             file_format = get_format(req)
             mutable_type = get_mutable_type(file_format)

From e271cd1b649a31cf2b8e17dba8f613954674c2e2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Jan 2021 09:53:36 -0500
Subject: [PATCH 12/50] Another test passing on Python 3.

---
 src/allmydata/test/web/test_web.py | 12 ++++++------
 src/allmydata/web/common.py        |  2 +-
 src/allmydata/web/directory.py     |  2 +-
 src/allmydata/web/filenode.py      |  4 ++--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 5123d3efe..27023ad49 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -268,9 +268,9 @@ class FakeClient(_Client):  # type: ignore  # tahoe-lafs/ticket/3573
         self.introducer_furls = []
         self.introducer_clients = []
         self.stats_provider = FakeStatsProvider()
-        self._secret_holder = SecretHolder("lease secret", "convergence secret")
+        self._secret_holder = SecretHolder(b"lease secret", b"convergence secret")
         self.helper = None
-        self.convergence = "some random string"
+        self.convergence = b"some random string"
         self.storage_broker = StorageFarmBroker(
             permute_peers=True,
             tub_maker=None,
@@ -1562,7 +1562,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             def _just_the_etag(result):
                 data, response, headers = result
                 etag = headers.getRawHeaders('etag')[0]
-                if uri.startswith('URI:DIR'):
+                if uri.startswith(b'URI:DIR'):
                     self.failUnless(etag.startswith('DIR:'), etag)
                 return etag
             return d.addCallback(_just_the_etag)
@@ -1570,7 +1570,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # Check that etags work with immutable directories
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
-                      json.dumps(newkids))
+                       json.dumps(newkids).encode("ascii"))
         def _stash_immdir_uri(uri):
             self._immdir_uri = uri
             return uri
@@ -1598,13 +1598,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check_match)
 
         def _no_etag(uri, t):
-            target = "/uri/%s?t=%s" % (uri, t)
+            target = "/uri/%s?t=%s" % (unicode(uri, "ascii"), t)
             d = self.GET(target, return_response=True, followRedirect=True)
             d.addCallback(lambda data_code_headers:
                           self.failIf(data_code_headers[2].hasHeader("etag"), target))
             return d
         def _yes_etag(uri, t):
-            target = "/uri/%s?t=%s" % (uri, t)
+            target = "/uri/%s?t=%s" % (unicode(uri, "ascii"), t)
             d = self.GET(target, return_response=True, followRedirect=True)
             d.addCallback(lambda data_code_headers:
                           self.failUnless(data_code_headers[2].hasHeader("etag"), target))
diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index 4a06d4ffd..b642a0b28 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -186,7 +186,7 @@ def convert_children_json(nodemaker, children_json):
     children = {}
     if children_json:
         data = json.loads(children_json)
-        for (namex, (ctype, propdict)) in data.iteritems():
+        for (namex, (ctype, propdict)) in data.items():
             namex = unicode(namex)
             writecap = to_bytes(propdict.get("rw_uri"))
             readcap = to_bytes(propdict.get("ro_uri"))
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 2a24cfa3b..952dcbc23 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -224,7 +224,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         FIXED_OUTPUT_TYPES =  ["", "json", "uri", "readonly-uri"]
         if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
             si = self.node.get_storage_index()
-            if si and req.setETag('DIR:%s-%s' % (base32.b2a(si), t or "")):
+            if si and req.setETag(b'DIR:%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
                 return b""
 
         if not t:
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index b34048623..4efc5d63a 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -188,8 +188,8 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
             # if the client already has the ETag then we can
             # short-circuit the whole process.
             si = self.node.get_storage_index()
-            if si and req.setETag('%s-%s' % (base32.b2a(si), t or "")):
-                return ""
+            if si and req.setETag(b'%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
+                return b""
 
         if not t:
             # just get the contents

From 8493d70cab3d5eb86cba3f26ab2c6fbb9ebb916d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Jan 2021 11:01:50 -0500
Subject: [PATCH 13/50] Support byte keys in internal dictionaries.

---
 src/allmydata/test/test_util.py |  4 ++--
 src/allmydata/util/jsonbytes.py | 31 +++++++++++++++++++------------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py
index c556eb4b9..58de96d1c 100644
--- a/src/allmydata/test/test_util.py
+++ b/src/allmydata/test/test_util.py
@@ -493,10 +493,10 @@ class JSONBytes(unittest.TestCase):
     def test_encode_bytes(self):
         """BytesJSONEncoder can encode bytes."""
         data = {
-            b"hello": [1, b"cd"],
+            b"hello": [1, b"cd", {b"abc": 123}],
         }
         expected = {
-            u"hello": [1, u"cd"],
+            u"hello": [1, u"cd", {u"abc": 123}],
         }
         # Bytes get passed through as if they were UTF-8 Unicode:
         encoded = jsonbytes.dumps(data)
diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index 406a471a0..ab9d5fac0 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -13,20 +13,34 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-
 import json
 
 
+def _bytes_to_unicode(obj):
+    """Convert keys of dicts from bytes to unicode, recursively."""
+    if isinstance(obj, bytes):
+        return obj.decode("utf-8")
+    if isinstance(obj, dict):
+        new_obj = {}
+        for k, v in obj.items():
+            if isinstance(k, bytes):
+                k = k.decode("utf-8")
+            v = _bytes_to_unicode(v)
+            new_obj[k] = v
+        return new_obj
+    if isinstance(obj, (list, set, tuple)):
+        return [_bytes_to_unicode(i) for i in obj]
+    return obj
+
+
 class BytesJSONEncoder(json.JSONEncoder):
     """
     A JSON encoder than can also encode bytes.
 
     The bytes are assumed to be UTF-8 encoded Unicode strings.
     """
-    def default(self, o):
-        if isinstance(o, bytes):
-            return o.decode("utf-8")
-        return json.JSONEncoder.default(self, o)
+    def iterencode(self, o, **kwargs):
+        return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs)
 
 
 def dumps(obj, *args, **kwargs):
@@ -34,13 +48,6 @@ def dumps(obj, *args, **kwargs):
 
     The bytes are assumed to be UTF-8 encoded Unicode strings.
     """
-    if isinstance(obj, dict):
-        new_obj = {}
-        for k, v in obj.items():
-            if isinstance(k, bytes):
-                k = k.decode("utf-8")
-            new_obj[k] = v
-        obj = new_obj
     return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
 
 

From 5a2339ced31085ab7397d8db0dcf783a509b832f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Jan 2021 11:02:57 -0500
Subject: [PATCH 14/50] Another test passing on Python 3.

---
 src/allmydata/test/web/test_web.py | 18 +++++++++---------
 src/allmydata/web/common_py3.py    |  8 ++++++--
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 27023ad49..203585b42 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -212,7 +212,7 @@ class FakeDisplayableServer(StubServer):  # type: ignore  # tahoe-lafs/ticket/35
             "application-version": "1.0"
         }
     def get_permutation_seed(self):
-        return ""
+        return b""
     def get_announcement(self):
         return self.announcement
     def get_nickname(self):
@@ -579,15 +579,14 @@ class WebMixin(TimezoneMixin):
             if isinstance(value, tuple):
                 filename, value = value
                 form.append('Content-Disposition: form-data; name="%s"; '
-                            'filename="%s"' % (name, filename.encode("utf-8")))
+                            'filename="%s"' % (name, filename))
             else:
                 form.append('Content-Disposition: form-data; name="%s"' % name)
             form.append('')
-            if isinstance(value, unicode):
-                value = value.encode("utf-8")
-            else:
-                value = str(value)
-            assert isinstance(value, str)
+            if isinstance(value, bytes):
+                value = unicode(value, "utf-8")
+            if not isinstance(value, unicode):
+                value = unicode(value)
             form.append(value)
             form.append(sep)
         form[-1] += "--"
@@ -596,7 +595,7 @@ class WebMixin(TimezoneMixin):
         if fields:
             body = "\r\n".join(form) + "\r\n"
             headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
-        return (body, headers)
+        return (body.encode("utf-8"), headers)
 
     def POST(self, urlpath, **fields):
         body, headers = self.build_form(**fields)
@@ -3156,7 +3155,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_DIRURL_check(self):
         foo_url = self.public_url + "/foo"
         res = yield self.POST(foo_url, t="check")
-        self.failUnlessIn("Healthy :", res)
+        self.failUnlessIn(b"Healthy :", res)
 
         redir_url = "http://allmydata.org/TARGET"
         body, headers = self.build_form(t="check", when_done=redir_url)
@@ -3165,6 +3164,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(foo_url, t="check", return_to=redir_url)
+        res = unicode(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file/directory", res)
         self.failUnlessIn(redir_url, res)
diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py
index cde3924fd..8bf728b4b 100644
--- a/src/allmydata/web/common_py3.py
+++ b/src/allmydata/web/common_py3.py
@@ -40,8 +40,12 @@ def get_arg(req, argname, default=None, multiple=False):
     results = []
     if argname in req.args:
         results.extend(req.args[argname])
-    if req.fields and argname in req.fields:
-        results.append(req.fields[argname].value)
+    argname_unicode = unicode(argname, "utf-8")
+    if req.fields and argname_unicode in req.fields:
+        value = req.fields[argname_unicode].value
+        if isinstance(value, unicode):
+            value = value.encode("utf-8")
+        results.append(value)
     if multiple:
         return tuple(results)
     if results:

From 9f5e1381bf0fa1d8f29c7851fe3fb7a182d86158 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Jan 2021 12:40:07 -0500
Subject: [PATCH 15/50] Another passing Python 3 test.

---
 src/allmydata/test/common.py       |  2 +-
 src/allmydata/test/web/test_web.py | 20 +++++++++++---------
 src/allmydata/web/check_results.py |  2 +-
 src/allmydata/web/operations.py    |  5 +++--
 src/allmydata/webish.py            |  4 ++--
 5 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index c28db9191..8d50390e4 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -432,7 +432,7 @@ class FakeCHKFileNode(object):  # type: ignore # incomplete implementation
         return self.storage_index
 
     def check(self, monitor, verify=False, add_lease=False):
-        s = StubServer("\x00"*20)
+        s = StubServer(b"\x00"*20)
         r = CheckResults(self.my_uri, self.storage_index,
                          healthy=True, recoverable=True,
                          count_happiness=10,
diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 203585b42..8ed7de880 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -3178,7 +3178,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_DIRURL_check_and_repair(self):
         foo_url = self.public_url + "/foo"
         res = yield self.POST(foo_url, t="check", repair="true")
-        self.failUnlessIn("Healthy :", res)
+        self.failUnlessIn(b"Healthy :", res)
 
         redir_url = "http://allmydata.org/TARGET"
         body, headers = self.build_form(t="check", repair="true",
@@ -3187,6 +3187,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     method="post", data=body, headers=headers,
                                     code=http.FOUND)
         res = yield self.POST(foo_url, t="check", return_to=redir_url)
+        res = unicode(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file/directory", res)
         self.failUnlessIn(redir_url, res)
@@ -3195,9 +3196,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         quux_url = "/uri/%s" % urlquote(self._quux_txt_uri)
         d = self.POST(quux_url, t="check")
         def _check(res):
-            self.failUnlessIn("Healthy", res)
+            self.failUnlessIn(b"Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % self._quux_txt_uri)
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % unicode(self._quux_txt_uri, "utf-8"))
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check"))
         d.addCallback(_check)
@@ -3264,15 +3265,15 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         self.failUnlessReallyEqual(data["count-objects-healthy"], 11)
 
         res = yield self.get_operation_results(None, "123", "html")
-        self.failUnlessIn("Objects Checked: <span>11</span>", res)
-        self.failUnlessIn("Objects Healthy: <span>11</span>", res)
+        self.failUnlessIn(b"Objects Checked: <span>11</span>", res)
+        self.failUnlessIn(b"Objects Healthy: <span>11</span>", res)
         soup = BeautifulSoup(res, 'html5lib')
         assert_soup_has_favicon(self, soup)
 
         res = yield self.GET("/operations/123/")
         # should be the same as without the slash
-        self.failUnlessIn("Objects Checked: <span>11</span>", res)
-        self.failUnlessIn("Objects Healthy: <span>11</span>", res)
+        self.failUnlessIn(b"Objects Checked: <span>11</span>", res)
+        self.failUnlessIn(b"Objects Healthy: <span>11</span>", res)
         soup = BeautifulSoup(res, 'html5lib')
         assert_soup_has_favicon(self, soup)
 
@@ -3282,9 +3283,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         foo_si = self._foo_node.get_storage_index()
         foo_si_s = base32.b2a(foo_si)
-        res = yield self.GET("/operations/123/%s?output=JSON" % foo_si_s)
+        res = yield self.GET("/operations/123/%s?output=JSON" % unicode(foo_si_s, "ascii"))
         data = json.loads(res)
-        self.failUnlessEqual(data["storage-index"], foo_si_s)
+        self.failUnlessEqual(data["storage-index"], unicode(foo_si_s, "ascii"))
         self.failUnless(data["results"]["healthy"])
 
     def test_POST_DIRURL_deepcheck_and_repair(self):
@@ -4261,6 +4262,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                  http.FOUND,
                  http.TEMPORARY_REDIRECT,
                  ] if code is None else [code]
+        body = yield response.content()
         self.assertIn(response.code, codes)
         location = response.headers.getRawHeaders(b"location")[0]
         if target_location is not None:
diff --git a/src/allmydata/web/check_results.py b/src/allmydata/web/check_results.py
index 54130183b..e3de4a7ce 100644
--- a/src/allmydata/web/check_results.py
+++ b/src/allmydata/web/check_results.py
@@ -432,7 +432,7 @@ class DeepCheckResultsRenderer(MultiFormatResource):
             return CheckResultsRenderer(self._client,
                                         r.get_results_for_storage_index(si))
         except KeyError:
-            raise WebError("No detailed results for SI %s" % html.escape(name),
+            raise WebError("No detailed results for SI %s" % html.escape(str(name, "utf-8")),
                            http.NOT_FOUND)
 
     @render_exception
diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py
index ca6826169..6769fb044 100644
--- a/src/allmydata/web/operations.py
+++ b/src/allmydata/web/operations.py
@@ -1,3 +1,4 @@
+from past.builtins import unicode
 
 import time
 from hyperlink import (
@@ -106,7 +107,7 @@ class OphandleTable(resource.Resource, service.Service):
         (monitor, renderer, when_added) = self.handles[ophandle]
 
         t = get_arg(req, "t", "status")
-        if t == "cancel" and req.method == b"POST":
+        if t == b"cancel" and req.method == b"POST":
             monitor.cancel()
             # return the status anyways, but release the handle
             self._release_ophandle(ophandle)
@@ -151,7 +152,7 @@ class ReloadMixin(object):
     @renderer
     def refresh(self, req, tag):
         if self.monitor.is_finished():
-            return ""
+            return b""
         tag.attributes["http-equiv"] = "refresh"
         tag.attributes["content"] = str(self.REFRESH_TIME)
         return tag
diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index e90fa573a..c3965645a 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -211,7 +211,7 @@ class WebishServer(service.MultiService):
         # use to test ophandle expiration.
         self._operations = OphandleTable(clock)
         self._operations.setServiceParent(self)
-        self.root.putChild("operations", self._operations)
+        self.root.putChild(b"operations", self._operations)
 
         self.root.putChild(b"storage-plugins", StoragePlugins(client))
 
@@ -220,7 +220,7 @@ class WebishServer(service.MultiService):
         self.site = TahoeLAFSSite(tempdir, self.root)
         self.staticdir = staticdir # so tests can check
         if staticdir:
-            self.root.putChild("static", static.File(staticdir))
+            self.root.putChild(b"static", static.File(staticdir))
         if re.search(r'^\d', webport):
             webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
         # strports must be native strings.

From 64f3e1277e720321da8e22d598acf1e013c1d0a1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 27 Jan 2021 11:33:09 -0500
Subject: [PATCH 16/50] Replace eliot_logged_test with something ... else

The previous implementation relied on TestCase.addCleanup in an unreliable
way.

This implementation pushes the decoration logic in to the test method itself
on the premise that test methods that do weird stuff are better supported than
`run` methods that do weird stuff.

Also add some more tests for this behavior.
---
 src/allmydata/test/eliotutil.py      | 198 +++++++++++++--------------
 src/allmydata/test/test_eliotutil.py |  68 +++++++++
 2 files changed, 166 insertions(+), 100 deletions(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index 88b4ec90e..3579eb31f 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -6,17 +6,22 @@ Tools aimed at the interaction between tests and Eliot.
 # Can't use `builtins.str` because it's not JSON encodable:
 # `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
 from past.builtins import unicode as str
-from future.utils import PY3
+from future.utils import PY2
+from six import ensure_text
 
 __all__ = [
     "RUN_TEST",
     "EliotLoggedRunTest",
-    "eliot_logged_test",
 ]
 
+try:
+    from typing import Callable
+except ImportError:
+    pass
+
 from functools import (
-    wraps,
     partial,
+    wraps,
 )
 
 import attr
@@ -24,11 +29,11 @@ import attr
 from eliot import (
     ActionType,
     Field,
+    MemoryLogger,
 )
-from eliot.testing import capture_logging
-
-from twisted.internet.defer import (
-    maybeDeferred,
+from eliot.testing import (
+    swap_logger,
+    check_for_errors,
 )
 
 from ..util.jsonbytes import BytesJSONEncoder
@@ -48,92 +53,12 @@ RUN_TEST = ActionType(
 )
 
 
-def eliot_logged_test(f):
-    """
-    Decorate a test method to run in a dedicated Eliot action context.
-
-    The action will finish after the test is done (after the returned Deferred
-    fires, if a Deferred is returned).  It will note the name of the test
-    being run.
-
-    All messages emitted by the test will be validated.  They will still be
-    delivered to the global logger.
-    """
-    # A convenient, mutable container into which nested functions can write
-    # state to be shared among them.
-    class storage(object):
-        pass
-
-
-    # On Python 3, we want to use our custom JSON encoder when validating
-    # messages can be encoded to JSON:
-    if PY3:
-        capture = lambda f : capture_logging(None, encoder_=BytesJSONEncoder)(f)
-    else:
-        capture = lambda f : capture_logging(None)(f)
-
-    @wraps(f)
-    def run_and_republish(self, *a, **kw):
-        # Unfortunately the only way to get at the global/default logger...
-        # This import is delayed here so that we get the *current* default
-        # logger at the time the decorated function is run.
-        from eliot._output import _DEFAULT_LOGGER as default_logger
-
-        def republish():
-            # This is called as a cleanup function after capture_logging has
-            # restored the global/default logger to its original state.  We
-            # can now emit messages that go to whatever global destinations
-            # are installed.
-
-            # storage.logger.serialize() seems like it would make more sense
-            # than storage.logger.messages here.  However, serialize()
-            # explodes, seemingly as a result of double-serializing the logged
-            # messages.  I don't understand this.
-            for msg in storage.logger.messages:
-                default_logger.write(msg)
-
-            # And now that we've re-published all of the test's messages, we
-            # can finish the test's action.
-            storage.action.finish()
-
-        @capture
-        def run(self, logger):
-            # Record the MemoryLogger for later message extraction.
-            storage.logger = logger
-            # Give the test access to the logger as well.  It would be just
-            # fine to pass this as a keyword argument to `f` but implementing
-            # that now will give me conflict headaches so I'm not doing it.
-            self.eliot_logger = logger
-            return f(self, *a, **kw)
-
-        # Arrange for all messages written to the memory logger that
-        # `capture_logging` installs to be re-written to the global/default
-        # logger so they might end up in a log file somewhere, if someone
-        # wants.  This has to be done in a cleanup function (or later) because
-        # capture_logging restores the original logger in a cleanup function.
-        # We install our cleanup function here, before we call run, so that it
-        # runs *after* the cleanup function capture_logging installs (cleanup
-        # functions are a stack).
-        self.addCleanup(republish)
-
-        # Begin an action that should comprise all messages from the decorated
-        # test method.
-        with RUN_TEST(name=self.id()).context() as action:
-            # When the test method Deferred fires, the RUN_TEST action is
-            # done.  However, we won't have re-published the MemoryLogger
-            # messages into the global/default logger when this Deferred
-            # fires.  So we need to delay finishing the action until that has
-            # happened.  Record the action so we can do that.
-            storage.action = action
-
-            # Support both Deferred-returning and non-Deferred-returning
-            # tests.
-            d = maybeDeferred(run, self)
-
-            # Let the test runner do its thing.
-            return d
-
-    return run_and_republish
+# On Python 3, we want to use our custom JSON encoder when validating messages
+# can be encoded to JSON:
+if PY2:
+    _memory_logger = MemoryLogger
+else:
+    _memory_logger = lambda: MemoryLogger(encoder=BytesJSONEncoder)
 
 
 @attr.s
@@ -174,10 +99,83 @@ class EliotLoggedRunTest(object):
     def id(self):
         return self.case.id()
 
-    @eliot_logged_test
-    def run(self, result=None):
-        return self._run_tests_with_factory(
-            self.case,
-            self.handlers,
-            self.last_resort,
-        ).run(result)
+    def run(self, result):
+        """
+        Run the test case in the context of a distinct Eliot action.
+
+        The action will finish after the test is done.  It will note the name of
+        the test being run.
+
+        All messages emitted by the test will be validated.  They will still be
+        delivered to the global logger.
+        """
+        # The idea here is to decorate the test method itself so that all of
+        # the extra logic happens at the point where test/application logic is
+        # expected to be.  This `run` method is more like test infrastructure
+        # and things do not go well when we add too much extra behavior here.
+        # For example, exceptions raised here often just kill the whole
+        # runner.
+
+        # So, grab the test method.
+        name = self.case._testMethodName
+        original = getattr(self.case, name)
+        try:
+            # Patch in a decorated version of it.
+            setattr(self.case, name, with_logging(ensure_text(self.case.id()), original))
+            # Then use the rest of the machinery to run it.
+            return self._run_tests_with_factory(
+                self.case,
+                self.handlers,
+                self.last_resort,
+            ).run(result)
+        finally:
+            # Clean up the patching for idempotency or something.
+            delattr(self.case, name)
+
+
+def with_logging(
+        test_id,      # type: unicode
+        test_method,  # type: Callable
+):
+    """
+    Decorate a test method with additional log-related behaviors.
+
+    1. The test method will run in a distinct Eliot action.
+    2. Typed log messages will be validated.
+    3. Logged tracebacks will be added as errors.
+
+    :param test_id: The full identifier of the test being decorated.
+    :param test_method: The method itself.
+    """
+    @wraps(test_method)
+    def run_with_logging(*args, **kwargs):
+        validating_logger = _memory_logger()
+        original = swap_logger(None)
+        try:
+            swap_logger(_TwoLoggers(original, validating_logger))
+            with RUN_TEST(name=test_id):
+                try:
+                    return test_method(*args, **kwargs)
+                finally:
+                    check_for_errors(validating_logger)
+        finally:
+            swap_logger(original)
+    return run_with_logging
+
+
+class _TwoLoggers(object):
+    """
+    Log to two loggers.
+
+    A single logger can have multiple destinations so this isn't typically a
+    useful thing to do.  However, MemoryLogger has inline validation instead
+    of destinations.  That means this *is* useful to simultaneously write to
+    the normal places and validate all written log messages.
+    """
+    def __init__(self, a, b):
+        self._a = a
+        self._b = b
+
+    def write(self, *args, **kwargs):
+        self._a.write(*args, **kwargs)
+        self._b.write(*args, **kwargs)
diff --git a/src/allmydata/test/test_eliotutil.py b/src/allmydata/test/test_eliotutil.py
index 0073a7675..aca677323 100644
--- a/src/allmydata/test/test_eliotutil.py
+++ b/src/allmydata/test/test_eliotutil.py
@@ -18,17 +18,25 @@ if PY2:
 from sys import stdout
 import logging
 
+from unittest import (
+    skip,
+)
+
 from fixtures import (
     TempDir,
 )
 from testtools import (
     TestCase,
 )
+from testtools import (
+    TestResult,
+)
 from testtools.matchers import (
     Is,
     IsInstance,
     MatchesStructure,
     Equals,
+    HasLength,
     AfterPreprocessing,
 )
 from testtools.twistedsupport import (
@@ -38,12 +46,16 @@ from testtools.twistedsupport import (
 
 from eliot import (
     Message,
+    MessageType,
+    fields,
     FileDestination,
+    MemoryLogger,
 )
 from eliot.twisted import DeferredContext
 from eliot.testing import (
     capture_logging,
     assertHasAction,
+    swap_logger,
 )
 
 from twisted.internet.defer import (
@@ -173,6 +185,62 @@ class EliotLoggingTests(TestCase):
             ),
         )
 
+    def test_validation_failure(self):
+        """
+        If a test emits a log message that fails validation then an error is added
+        to the result.
+        """
+        # Make sure we preserve the original global Eliot state.
+        original = swap_logger(MemoryLogger())
+        self.addCleanup(lambda: swap_logger(original))
+
+        class ValidationFailureProbe(SyncTestCase):
+            def test_bad_message(self):
+                # This message does not validate because "Hello" is not an
+                # int.
+                MSG = MessageType("test:eliotutil", fields(foo=int))
+                MSG(foo="Hello").write()
+
+        result = TestResult()
+        case = ValidationFailureProbe("test_bad_message")
+        case.run(result)
+
+        self.assertThat(
+            result.errors,
+            HasLength(1),
+        )
+
+    def test_skip_cleans_up(self):
+        """
+        After a skipped test the global Eliot logging state is restored.
+        """
+        # Save the logger that's active before we do anything so that we can
+        # restore it later.  Also install another logger so we can compare it
+        # to the active logger later.
+        expected = MemoryLogger()
+        original = swap_logger(expected)
+
+        # Restore it, whatever else happens.
+        self.addCleanup(lambda: swap_logger(original))
+
+        class SkipProbe(SyncTestCase):
+            @skip("It's a skip test.")
+            def test_skipped(self):
+                pass
+
+        case = SkipProbe("test_skipped")
+        case.run()
+
+        # Retrieve the logger that's active now that the skipped test is done
+        # so we can check it against the expected value.
+        actual = swap_logger(MemoryLogger())
+        self.assertThat(
+            actual,
+            Is(expected),
+        )
+
+
+
 class LogCallDeferredTests(TestCase):
     """
     Tests for ``log_call_deferred``.

From 740fe9fef7f83bb4de0bdbf79f33560d9ca909dc Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 27 Jan 2021 11:36:53 -0500
Subject: [PATCH 17/50] news fragment

---
 newsfragments/3600.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3600.minor

diff --git a/newsfragments/3600.minor b/newsfragments/3600.minor
new file mode 100644
index 000000000..e69de29bb

From c1391159fac566f6125dfdb98e2b98d7a044ca77 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 27 Jan 2021 12:06:25 -0500
Subject: [PATCH 18/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 13 ++++++++++---
 src/allmydata/web/common_py3.py    |  8 +++++++-
 src/allmydata/web/directory.py     |  2 +-
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 8ed7de880..008593cf2 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -634,7 +634,10 @@ class WebMixin(TimezoneMixin):
                                        getattr(res.value, "response", ""),
                                        which))
                 if response_substring:
-                    self.failUnlessIn(response_substring, unicode(res.value.response, "utf-8"),
+                    response = res.value.response
+                    if isinstance(response, bytes):
+                        response = unicode(response, "utf-8")
+                    self.failUnlessIn(response_substring, response,
                                       "'%s' not in '%s' for test '%s'" % \
                                       (response_substring, res.value.response,
                                        which))
@@ -2209,11 +2212,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _got_json(data):
             self.failUnlessReallyEqual(data["finished"], True)
             size = data["size"]
-            self.failUnless(size > 1000)
+            # Match calculation of text value size below:
+            self.failUnless(
+                size.get("size-directories", 0) + size.get("size-mutable-files", 0) +
+                size.get("size-immutable-files", 0) > 1000)
         d.addCallback(_got_json)
         d.addCallback(self.get_operation_results, "126", "text")
         def _got_text(res):
-            mo = re.search(r'^size: (\d+)$', res, re.M)
+            mo = re.search(br'^size: (\d+)$', res, re.M)
             self.failUnless(mo, res)
             size = int(mo.group(1))
             # with directories, the size varies.
@@ -3311,6 +3317,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "124", "html")
         def _check_html(res):
+            res = unicode(res, "utf-8")
             self.failUnlessIn("Objects Checked: <span>11</span>", res)
 
             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py
index 8bf728b4b..439d346fa 100644
--- a/src/allmydata/web/common_py3.py
+++ b/src/allmydata/web/common_py3.py
@@ -83,7 +83,13 @@ class MultiFormatResource(resource.Resource, object):
         if isinstance(t, bytes):
             t = unicode(t, "ascii")
         renderer = self._get_renderer(t)
-        return renderer(req)
+        result = renderer(req)
+        # On Python 3, json.dumps() returns Unicode for example, but
+        # twisted.web expects bytes. Instead of updating every single render
+        # method, just handle Unicode one time here.
+        if isinstance(result, unicode):
+            result = result.encode("utf-8")
+        return result
 
     def _get_renderer(self, fmt):
         """
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 952dcbc23..7570a5724 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -1290,7 +1290,7 @@ class DeepSizeResults(MultiFormatResource):
                      + stats.get("size-mutable-files", 0)
                      + stats.get("size-directories", 0))
             output += "size: %d\n" % total
-        return output
+        return output.encode("utf-8")
     render_TEXT = render_HTML
 
     def render_JSON(self, req):

From 4f56ce436ab69dc1704339017fe0fd714c4229a1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 27 Jan 2021 12:39:28 -0500
Subject: [PATCH 19/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 10 +++++-----
 src/allmydata/web/directory.py     | 12 ++++++------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 008593cf2..4996d841e 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -2163,7 +2163,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             soup = BeautifulSoup(manifest, 'html5lib')
             assert_soup_has_text(self, soup, "Manifest of SI=")
             assert_soup_has_text(self, soup, "sub")
-            assert_soup_has_text(self, soup, self._sub_uri)
+            assert_soup_has_text(self, soup, unicode(self._sub_uri, "ascii"))
             assert_soup_has_text(self, soup, "sub/baz.txt")
             assert_soup_has_favicon(self, soup)
         d.addCallback(_got_html)
@@ -2176,8 +2176,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_got_html)
         d.addCallback(getman, "text")
         def _got_text(manifest):
-            self.failUnlessIn("\nsub " + self._sub_uri + "\n", manifest)
-            self.failUnlessIn("\nsub/baz.txt URI:CHK:", manifest)
+            self.failUnlessIn(b"\nsub " + self._sub_uri + b"\n", manifest)
+            self.failUnlessIn(b"\nsub/baz.txt URI:CHK:", manifest)
         d.addCallback(_got_text)
         d.addCallback(getman, "JSON")
         def _got_json(res):
@@ -2267,8 +2267,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_DIRURL_stream_manifest(self):
         d = self.POST(self.public_url + "/foo?t=stream-manifest")
         def _check(res):
-            self.failUnless(res.endswith("\n"))
-            units = [json.loads(t) for t in res[:-1].split("\n")]
+            self.failUnless(res.endswith(b"\n"))
+            units = [json.loads(t) for t in res[:-1].split(b"\n")]
             self.failUnlessReallyEqual(len(units), 10)
             self.failUnlessEqual(units[-1]["type"], "stats")
             first = units[0]
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 7570a5724..40f4fcf9a 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -1144,8 +1144,8 @@ def _slashify_path(path):
     in it
     """
     if not path:
-        return ""
-    return "/".join([p.encode("utf-8") for p in path])
+        return b""
+    return b"/".join([p.encode("utf-8") for p in path])
 
 
 def _cap_to_link(root, path, cap):
@@ -1234,10 +1234,10 @@ class ManifestResults(MultiFormatResource, ReloadMixin):
         req.setHeader("content-type", "text/plain")
         lines = []
         is_finished = self.monitor.is_finished()
-        lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
+        lines.append(b"finished: " + {True: b"yes", False: b"no"}[is_finished])
         for path, cap in self.monitor.get_status()["manifest"]:
-            lines.append(_slashify_path(path) + " " + cap)
-        return "\n".join(lines) + "\n"
+            lines.append(_slashify_path(path) + b" " + cap)
+        return b"\n".join(lines) + b"\n"
 
     def render_JSON(self, req):
         req.setHeader("content-type", "text/plain")
@@ -1315,7 +1315,7 @@ class DeepStatsResults(Resource, object):
         req.setHeader("content-type", "text/plain")
         s = self.monitor.get_status().copy()
         s["finished"] = self.monitor.is_finished()
-        return json.dumps(s, indent=1)
+        return json.dumps(s, indent=1).encode("utf-8")
 
 
 @implementer(IPushProducer)

From 175a3ca4bbc6166918b3ef93db99c35573f471e2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 27 Jan 2021 12:45:36 -0500
Subject: [PATCH 20/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 12 +++++++-----
 src/allmydata/web/filenode.py      |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 4996d841e..8dc22a4e1 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -2363,7 +2363,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_NEWDIRURL_bad_format(self):
         url = (self.webish_url + self.public_url +
                "/foo/newdir?t=mkdir&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="post", data="")
 
     def test_POST_NEWDIRURL_emptyname(self):
@@ -3121,7 +3121,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_FILEURL_check(self):
         bar_url = self.public_url + "/foo/bar.txt"
         res = yield self.POST(bar_url, t="check")
-        self.failUnlessIn("Healthy :", res)
+        self.failUnlessIn(b"Healthy :", res)
 
         redir_url = "http://allmydata.org/TARGET"
         body, headers = self.build_form(t="check", when_done=redir_url)
@@ -3130,6 +3130,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(bar_url, t="check", return_to=redir_url)
+        res = unicode(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file", res)
         self.failUnlessIn(redir_url, res)
@@ -3143,7 +3144,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_FILEURL_check_and_repair(self):
         bar_url = self.public_url + "/foo/bar.txt"
         res = yield self.POST(bar_url, t="check", repair="true")
-        self.failUnlessIn("Healthy :", res)
+        self.failUnlessIn(b"Healthy :", res)
 
         redir_url = "http://allmydata.org/TARGET"
         body, headers = self.build_form(t="check", repair="true",
@@ -3153,6 +3154,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(bar_url, t="check", return_to=redir_url)
+        res = unicode(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file", res)
         self.failUnlessIn(redir_url, res)
@@ -3214,9 +3216,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         quux_url = "/uri/%s" % urlquote(self._quux_txt_uri)
         d = self.POST(quux_url, t="check", repair="true")
         def _check(res):
-            self.failUnlessIn("Healthy", res)
+            self.failUnlessIn(b"Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % self._quux_txt_uri)
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % unicode(self._quux_txt_uri, "ascii"))
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check", repair="true"))
         d.addCallback(_check)
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index 4efc5d63a..d050fb8a0 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -309,7 +309,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
                 assert self.parentnode and self.name
                 d = self.replace_me_with_a_formpost(req, self.client, replace)
         else:
-            raise WebError("POST to file: bad t=%s" % t)
+            raise WebError("POST to file: bad t=%s" % unicode(t, "ascii"))
 
         return handle_when_done(req, d)
 

From ea10c7c937b90be7f401ae58cc14f5ade73c5f16 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 27 Jan 2021 15:20:38 -0500
Subject: [PATCH 21/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 17 ++++++++++-------
 src/allmydata/web/directory.py     |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 8dc22a4e1..7dc576bcd 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1,6 +1,7 @@
 from __future__ import print_function
 
 from past.builtins import unicode
+from six import ensure_binary
 
 import os.path, re, time
 import treq
@@ -603,6 +604,8 @@ class WebMixin(TimezoneMixin):
 
     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
         url = self.webish_url + urlpath
+        if isinstance(body, unicode):
+            body = body.encode("utf-8")
         return do_http("POST", url, allow_redirects=followRedirect,
                        headers=headers, data=body)
 
@@ -1572,7 +1575,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # Check that etags work with immutable directories
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2(self.public_url + "/foo/newdir?t=mkdir-immutable",
-                       json.dumps(newkids).encode("ascii"))
+                       json.dumps(newkids))
         def _stash_immdir_uri(uri):
             self._immdir_uri = uri
             return uri
@@ -2440,8 +2443,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         (newkids, caps) = self._create_initial_children()
         url = (self.webish_url + self.public_url +
                "/foo/newdir?t=mkdir-with-children&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
-                                   method="post", data=json.dumps(newkids))
+        yield self.assertHTTPError(url, 400, "Unknown format:",
+                                   method="post", data=json.dumps(newkids).encode("utf-8"))
 
     def test_POST_NEWDIRURL_immutable(self):
         (newkids, caps) = self._create_immutable_children()
@@ -2653,8 +2656,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check(child):
             self.failUnless(child.is_unknown() or child.is_readonly())
             self.failUnlessReallyEqual(child.get_write_uri(), None)
-            self.failUnlessReallyEqual(child.get_uri(), expected_uri.strip())
-            self.failUnlessReallyEqual(child.get_readonly_uri(), expected_uri.strip())
+            self.failUnlessReallyEqual(child.get_uri(), ensure_binary(expected_uri.strip()))
+            self.failUnlessReallyEqual(child.get_readonly_uri(), ensure_binary(expected_uri.strip()))
         d.addCallback(_check)
         return d
 
@@ -2663,8 +2666,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = node.get_child_at_path(name)
         def _check(child):
             self.failUnless(child.is_unknown() or not child.is_readonly())
-            self.failUnlessReallyEqual(child.get_uri(), got_uri.strip())
-            self.failUnlessReallyEqual(child.get_write_uri(), got_uri.strip())
+            self.failUnlessReallyEqual(child.get_uri(), ensure_binary(got_uri.strip()))
+            self.failUnlessReallyEqual(child.get_write_uri(), ensure_binary(got_uri.strip()))
             expected_ro_uri = self._make_readonly(got_uri)
             if expected_ro_uri:
                 self.failUnlessReallyEqual(child.get_readonly_uri(), expected_ro_uri.strip())
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 40f4fcf9a..50731cf39 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -436,8 +436,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             # a slightly confusing error message if someone does a POST
             # without a name= field. For our own HTML this isn't a big
             # deal, because we create the 'unlink' POST buttons ourselves.
-            name = ''
-        charset = get_arg(req, "_charset", "utf-8")
+            name = b''
+        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
         name = name.decode(charset)
         d = self.node.delete(name)
         d.addCallback(lambda res: "thing unlinked")

From 66688423564469469365bcb8a86d92bb2cf84d12 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 27 Jan 2021 15:43:44 -0500
Subject: [PATCH 22/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 26 +++++++++++++-------------
 src/allmydata/web/directory.py     |  6 +++---
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 7dc576bcd..b738f5673 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1749,7 +1749,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_NEWFILEURL_unlinked_bad_format(self):
         contents = self.NEWFILE_CONTENTS * 300000
         yield self.assertHTTPError(self.webish_url + "/uri?format=foo", 400,
-                                   "Unknown format: foo",
+                                   "Unknown format:",
                                    method="put", data=contents)
 
     def test_PUT_NEWFILEURL_range_bad(self):
@@ -2333,7 +2333,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_NEWDIRURL_bad_format(self):
         url = (self.webish_url + self.public_url +
                "/foo/newdir=?t=mkdir&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="put", data="")
 
     def test_POST_NEWDIRURL(self):
@@ -2567,7 +2567,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
         url = (self.webish_url + self.public_url +
                "/foo/subdir/newdir?t=mkdir&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="put", data="")
 
     def test_DELETE_DIRURL(self):
@@ -2843,7 +2843,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         url = self.webish_url + "/uri?t=upload&format=foo"
         body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
         yield self.assertHTTPError(url, 400,
-                                   "Unknown format: foo",
+                                   "Unknown format:",
                                    method="post", data=body, headers=headers)
 
     def test_POST_upload_format(self):
@@ -2878,7 +2878,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_upload_bad_format(self):
         url = self.webish_url + self.public_url + "/foo?t=upload&format=foo"
         body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="post", data=body, headers=headers)
 
     def test_POST_upload_mutable(self):
@@ -3373,7 +3373,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_mkdir_bad_format(self):
         url = (self.webish_url + self.public_url +
                "/foo?t=mkdir&name=newdir&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="post")
 
     def test_POST_mkdir_initial_children(self):
@@ -3425,8 +3425,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         (newkids, caps) = self._create_initial_children()
         url = (self.webish_url + self.public_url +
                "/foo?t=mkdir-with-children&name=newdir&format=foo")
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
-                                   method="post", data=json.dumps(newkids))
+        yield self.assertHTTPError(url, 400, "Unknown format:",
+                                   method="post", data=json.dumps(newkids).encode("utf-8"))
 
     def test_POST_mkdir_immutable(self):
         (newkids, caps) = self._create_immutable_children()
@@ -3504,7 +3504,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     @inlineCallbacks
     def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
         url = self.webish_url + self.public_url + "/uri?t=mkdir&format=foo"
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="post")
 
     def test_POST_mkdir_no_parentdir_noredirect2(self):
@@ -3606,7 +3606,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         (newkids, caps) = self._create_initial_children()
         d = self.POST2("/uri?t=mkdir-with-children", json.dumps(newkids))
         def _after_mkdir(res):
-            self.failUnless(res.startswith("URI:DIR"), res)
+            self.failUnless(res.startswith(b"URI:DIR"), res)
             n = self.s.create_node_from_uri(res)
             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
             d2.addCallback(lambda ign:
@@ -3657,7 +3657,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         (newkids, caps) = self._create_immutable_children()
         d = self.POST2("/uri?t=mkdir-immutable", json.dumps(newkids))
         def _after_mkdir(res):
-            self.failUnless(res.startswith("URI:DIR"), res)
+            self.failUnless(res.startswith(b"URI:DIR"), res)
             n = self.s.create_node_from_uri(res)
             d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
             d2.addCallback(lambda ign:
@@ -4446,7 +4446,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_NEWFILEURL_bad_format(self):
         new_contents = self.NEWFILE_CONTENTS * 300000
         url = self.webish_url + self.public_url + "/foo/foo.txt?format=foo"
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="put", data=new_contents)
 
     def test_PUT_NEWFILEURL_uri_replace(self):
@@ -4579,7 +4579,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     @inlineCallbacks
     def test_PUT_mkdir_bad_format(self):
         url = self.webish_url + "/uri?t=mkdir&format=foo"
-        yield self.assertHTTPError(url, 400, "Unknown format: foo",
+        yield self.assertHTTPError(url, 400, "Unknown format:",
                                    method="put", data="")
 
     def test_POST_check(self):
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 50731cf39..3c8b50b04 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -364,7 +364,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         return d
 
     def _POST_upload(self, req):
-        charset = get_arg(req, "_charset", "utf-8")
+        charset = unicode(get_arg(req, "_charset", b"utf-8"))
         contents = req.fields["file"]
         assert contents.filename is None or isinstance(contents.filename, str)
         name = get_arg(req, "name")
@@ -413,7 +413,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         name = get_arg(req, "name")
         if not name:
             raise WebError("set-uri requires a name")
-        charset = get_arg(req, "_charset", "utf-8")
+        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
         name = name.decode(charset)
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
@@ -453,7 +453,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         return self._POST_relink(req)
 
     def _POST_relink(self, req):
-        charset = get_arg(req, "_charset", "utf-8")
+        charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
         replace = parse_replace_arg(get_arg(req, "replace", "true"))
 
         from_name = get_arg(req, "from_name")

From 2007323baa0248b80369630d5358a39bf9eabe58 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 28 Jan 2021 11:19:09 -0500
Subject: [PATCH 23/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 40 +++++++++++++++---------------
 src/allmydata/web/unlinked.py      | 12 ++++-----
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index b738f5673..e15806938 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -3521,7 +3521,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         url = self.webish_url + "/uri?t=mkdir&redirect_to_result=true"
         target = yield self.shouldRedirectTo(url, None, method="post",
                                              code=http.SEE_OTHER)
-        target = urlunquote(target)
+        target = urlunquote(unicode(target, "ascii"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     @inlineCallbacks
@@ -3531,7 +3531,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                              method="post",
                                              data=body, headers=headers,
                                              code=http.SEE_OTHER)
-        target = urlunquote(target)
+        target = urlunquote(unicode(target, "ascii"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def _make_readonly(self, u):
@@ -3643,7 +3643,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         yield self.assertHTTPError(url, 400,
                                    "t=mkdir does not accept children=, "
                                    "try t=mkdir-with-children instead",
-                                   method="post", data=json.dumps(newkids))
+                                   method="post", data=json.dumps(newkids).encode("utf-8"))
 
     @inlineCallbacks
     def test_POST_noparent_bad(self):
@@ -4033,7 +4033,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_dir=self.public_root.get_uri() + "/foo/sub")
+                      to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4047,7 +4047,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_new_name(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="wibble.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+                      to_name="wibble.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4063,7 +4063,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_replace(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
@@ -4078,7 +4078,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "There was already a child by that name, and you asked me to not replace it",
                              self.POST, self.public_url + "/foo", t="relink",
                              replace="false", from_name="bar.txt",
-                             to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+                             to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
@@ -4090,7 +4090,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_no_replace_explicitly_same_link(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       replace="false", from_name="bar.txt",
-                      to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
+                      to_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
@@ -4101,7 +4101,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_replace_only_files(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       replace="only-files", from_name="bar.txt",
-                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
@@ -4116,7 +4116,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "There was already a child by that name, and you asked me to not replace it.",
                              self.POST, self.public_url + "/foo", t="relink",
                              replace="only-files", from_name="bar.txt",
-                             to_name="sub", to_dir=self.public_root.get_uri() + "/foo")
+                             to_name="sub", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
@@ -4129,7 +4129,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "to_name= may not contain a slash",
                              self.POST, self.public_url + "/foo", t="relink",
                              from_name="bar.txt",
-                             to_name="slash/fail.txt", to_dir=self.public_root.get_uri() + "/foo/sub")
+                             to_name="slash/fail.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4143,13 +4143,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                        t="relink",
                                        from_name="nope/bar.txt",
                                        to_name="fail.txt",
-                                       to_dir=self.public_root.get_uri() + "/foo/sub"))
+                                       to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub"))
         return d
 
     def test_POST_relink_file_explicitly_same_link(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo")
+                      to_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
@@ -4170,7 +4170,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_same_dir(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="baz.txt", to_dir=self.public_root.get_uri() + "/foo")
+                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
@@ -4185,13 +4185,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.POST,
                              self.public_url + "/foo", t="relink",
                              replace="boogabooga", from_name="bar.txt",
-                             to_dir=self.public_root.get_uri() + "/foo/sub")
+                             to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         return d
 
     def test_POST_relink_file_multi_level(self):
         d = self.POST2(self.public_url + "/foo/sub/level2?t=mkdir", "")
         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
-                      from_name="bar.txt", to_dir=self.public_root.get_uri() + "/foo/sub/level2"))
+                      from_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub/level2"))
         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
@@ -4216,7 +4216,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                             "404 Not Found", "No such child: nopechucktesta",
                             self.POST, self.public_url + "/foo", t="relink",
                             from_name="bar.txt",
-                            to_dir=self.public_root.get_uri() + "/nopechucktesta")
+                            to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/nopechucktesta")
         return d
 
     def test_POST_relink_file_into_file(self):
@@ -4224,7 +4224,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "400 Bad Request", "to_dir is not a directory",
                              self.POST, self.public_url + "/foo", t="relink",
                              from_name="bar.txt",
-                             to_dir=self.public_root.get_uri() + "/foo/baz.txt")
+                             to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/baz.txt")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
         d.addCallback(self.failUnlessIsBazDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
@@ -4248,10 +4248,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_dir(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_dir=self.public_root.get_uri() + "/foo/empty")
+                      to_dir=unicode(self.public_root.get_uri(), "ascii") + "/foo/empty")
         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
                       t="relink", from_name="empty",
-                      to_dir=self.public_root.get_uri() + "/foo/sub"))
+                      to_dir=unicode(self.public_root.get_uri(), "ascii") + "/foo/sub"))
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"empty"))
         d.addCallback(lambda res:
diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py
index e420a0371..55681c9c9 100644
--- a/src/allmydata/web/unlinked.py
+++ b/src/allmydata/web/unlinked.py
@@ -1,5 +1,5 @@
 
-import urllib
+from urllib.parse import quote as urlquote
 
 from twisted.web import http
 from twisted.internet import defer
@@ -66,7 +66,7 @@ def POSTUnlinkedCHK(req, client):
         # usual upload-results page
         def _done(upload_results, redir_to):
             if "%(uri)s" in redir_to:
-                redir_to = redir_to.replace("%(uri)s", urllib.quote(upload_results.get_uri()))
+                redir_to = redir_to.replace("%(uri)s", urlquote(upload_results.get_uri()))
             return url_for_string(req, redir_to)
         d.addCallback(_done, when_done)
     else:
@@ -119,7 +119,7 @@ class UploadResultsElement(status.UploadResultsRendererMixin):
         d = self.upload_results()
         d.addCallback(lambda res:
                       tags.a("/uri/" + res.get_uri(),
-                             href="/uri/" + urllib.quote(res.get_uri())))
+                             href="/uri/" + urlquote(res.get_uri())))
         return d
 
 
@@ -158,7 +158,7 @@ def POSTUnlinkedCreateDirectory(req, client):
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):
-            new_url = "uri/" + urllib.quote(res.get_uri())
+            new_url = "uri/" + urlquote(res.get_uri())
             req.setResponseCode(http.SEE_OTHER) # 303
             req.setHeader('location', new_url)
             return ''
@@ -176,7 +176,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client):
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):
-            new_url = "uri/" + urllib.quote(res.get_uri())
+            new_url = "uri/" + urlquote(res.get_uri())
             req.setResponseCode(http.SEE_OTHER) # 303
             req.setHeader('location', new_url)
             return ''
@@ -194,7 +194,7 @@ def POSTUnlinkedCreateImmutableDirectory(req, client):
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):
-            new_url = "uri/" + urllib.quote(res.get_uri())
+            new_url = "uri/" + urlquote(res.get_uri())
             req.setResponseCode(http.SEE_OTHER) # 303
             req.setHeader('location', new_url)
             return ''

From e5a7ea97c76b8b1bed4c660a00d2863a265a1dfd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 28 Jan 2021 11:44:16 -0500
Subject: [PATCH 24/50] More tests passing on Python 3.

---
 src/allmydata/test/web/test_web.py | 12 ++++++------
 src/allmydata/web/directory.py     | 20 ++++++++++++++------
 src/allmydata/web/unlinked.py      |  5 +++--
 3 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index e15806938..6527d5dcd 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -2817,7 +2817,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 else:
                     # for immutable, it returns an "upload results page", and
                     # the filecap is buried inside
-                    line = [l for l in results.split("\n") if "URI: " in l][0]
+                    line = [l for l in results.split(b"\n") if b"URI: " in l][0]
                     mo = re.search(r'<span>([^<]+)</span>', line)
                     filecap = mo.group(1)
                 self.failUnless(filecap.startswith(uri_prefix),
@@ -2868,10 +2868,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             return d
 
         d = defer.succeed(None)
-        d.addCallback(_check_upload, "chk", "URI:CHK")
-        d.addCallback(_check_upload, "sdmf", "URI:SSK", self._foo_node)
-        d.addCallback(_check_upload, "mdmf", "URI:MDMF")
-        d.addCallback(_check_upload, "MDMF", "URI:MDMF")
+        d.addCallback(_check_upload, "chk", b"URI:CHK")
+        d.addCallback(_check_upload, "sdmf", b"URI:SSK", self._foo_node)
+        d.addCallback(_check_upload, "mdmf", b"URI:MDMF")
+        d.addCallback(_check_upload, "MDMF", b"URI:MDMF")
         return d
 
     @inlineCallbacks
@@ -3776,7 +3776,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         contents10, n10, newuri10 = self.makefile(10)
         contents11, n11, newuri11 = self.makefile(11)
 
-        reqbody = """{
+        reqbody = b"""{
                      "atomic_added_1": [ "filenode", { "rw_uri": "%s",
                                                 "size": 0,
                                                 "metadata": {
diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index 3c8b50b04..03194b2a3 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -1,3 +1,11 @@
+"""
+TODO: When porting to Python 3, the filename handling logic seems wrong.  On
+Python 3 filename will _already_ be correctly decoded.  So only decode if it's
+bytes.
+
+Also there's a lot of code duplication I think.
+"""
+
 from past.builtins import unicode
 
 from urllib.parse import quote as url_quote
@@ -364,7 +372,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         return d
 
     def _POST_upload(self, req):
-        charset = unicode(get_arg(req, "_charset", b"utf-8"))
+        charset = unicode(get_arg(req, "_charset", b"utf-8"), "utf-8")
         contents = req.fields["file"]
         assert contents.filename is None or isinstance(contents.filename, str)
         name = get_arg(req, "name")
@@ -374,8 +382,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
         if not name:
             # this prohibts empty, missing, and all-whitespace filenames
             raise WebError("upload requires a name")
-        assert isinstance(name, str)
-        name = name.decode(charset)
+        if isinstance(name, bytes):
+            name = name.decode(charset)
         if "/" in name:
             raise WebError("name= may not contain a slash", http.BAD_REQUEST)
         assert isinstance(name, unicode)
@@ -624,14 +632,14 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
             # TODO test handling of bad JSON
             raise
         cs = {}
-        for name, (file_or_dir, mddict) in children.iteritems():
+        for name, (file_or_dir, mddict) in children.items():
             name = unicode(name) # json returns str *or* unicode
             writecap = mddict.get('rw_uri')
             if writecap is not None:
-                writecap = str(writecap)
+                writecap = writecap.encode("utf-8")
             readcap = mddict.get('ro_uri')
             if readcap is not None:
-                readcap = str(readcap)
+                readcap = readcap.encode("utf-8")
             cs[name] = (writecap, readcap, mddict.get('metadata'))
         d = self.node.set_children(cs, replace)
         d.addCallback(lambda res: "Okay so I did it.")
diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py
index 55681c9c9..2e6495957 100644
--- a/src/allmydata/web/unlinked.py
+++ b/src/allmydata/web/unlinked.py
@@ -1,3 +1,4 @@
+from past.builtins import unicode
 
 from urllib.parse import quote as urlquote
 
@@ -118,8 +119,8 @@ class UploadResultsElement(status.UploadResultsRendererMixin):
     def download_link(self, req, tag):
         d = self.upload_results()
         d.addCallback(lambda res:
-                      tags.a("/uri/" + res.get_uri(),
-                             href="/uri/" + urlquote(res.get_uri())))
+                      tags.a("/uri/" + unicode(res.get_uri(), "utf-8"),
+                             href="/uri/" + urlquote(unicode(res.get_uri(), "utf-8"))))
         return d
 
 

From 90a47bb859cfd608495f8667aaa44895b21c0104 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 28 Jan 2021 12:00:15 -0500
Subject: [PATCH 25/50] WIP

---
 src/allmydata/test/web/test_web.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 6527d5dcd..4ef9405e5 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -2818,11 +2818,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                     # for immutable, it returns an "upload results page", and
                     # the filecap is buried inside
                     line = [l for l in results.split(b"\n") if b"URI: " in l][0]
-                    mo = re.search(r'<span>([^<]+)</span>', line)
+                    mo = re.search(br'<span>([^<]+)</span>', line)
                     filecap = mo.group(1)
                 self.failUnless(filecap.startswith(uri_prefix),
                                 (uri_prefix, filecap))
-                return self.GET("/uri/%s?t=json" % filecap)
+                return self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8"))
             d.addCallback(_got_results)
             def _got_json(raw):
                 data = json.loads(raw)
@@ -2832,10 +2832,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             d.addCallback(_got_json)
             return d
         d = defer.succeed(None)
-        d.addCallback(_check_upload_unlinked, "chk", "URI:CHK")
-        d.addCallback(_check_upload_unlinked, "CHK", "URI:CHK")
-        d.addCallback(_check_upload_unlinked, "sdmf", "URI:SSK")
-        d.addCallback(_check_upload_unlinked, "mdmf", "URI:MDMF")
+        d.addCallback(_check_upload_unlinked, "chk", b"URI:CHK")
+        d.addCallback(_check_upload_unlinked, "CHK", b"URI:CHK")
+        d.addCallback(_check_upload_unlinked, "sdmf", b"URI:SSK")
+        d.addCallback(_check_upload_unlinked, "mdmf", b"URI:MDMF")
         return d
 
     @inlineCallbacks
@@ -2900,7 +2900,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_got)
 
         # now upload it again and make sure that the URI doesn't change
-        NEWER_CONTENTS = self.NEWFILE_CONTENTS + "newer\n"
+        NEWER_CONTENTS = self.NEWFILE_CONTENTS + b"newer\n"
         d.addCallback(lambda res:
                       self.POST(self.public_url + "/foo", t="upload",
                                 mutable="true",
@@ -2918,7 +2918,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_got2)
 
         # upload a second time, using PUT instead of POST
-        NEW2_CONTENTS = NEWER_CONTENTS + "overwrite with PUT\n"
+        NEW2_CONTENTS = NEWER_CONTENTS + b"overwrite with PUT\n"
         d.addCallback(lambda res:
                       self.PUT(self.public_url + "/foo/new.txt", NEW2_CONTENTS))
         d.addCallback(self.failUnlessURIMatchesRWChild, fn, u"new.txt")
@@ -2934,7 +2934,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                followRedirect=True))
         def _check_page(res):
             # TODO: assert more about the contents
-            self.failUnlessIn("SSK", res)
+            self.failUnlessIn(b"SSK", res)
             return res
         d.addCallback(_check_page)
 
@@ -3012,7 +3012,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.POST(self.public_url + "/foo", t="upload",
                       mutable="true",
                       file=("new.txt",
-                            "b" * (self.s.MUTABLE_SIZELIMIT+1))))
+                            b"b" * (self.s.MUTABLE_SIZELIMIT+1))))
         d.addErrback(self.dump_error)
         return d
 
@@ -3024,7 +3024,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # mutable files.
         d = self.POST(self.public_url + "/foo",
                       t="upload", mutable="true",
-                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
+                      file=("new.txt", b"b" * (self.s.MUTABLE_SIZELIMIT + 1)))
         return d
 
     def dump_error(self, f):

From 81edda80115c0bfcb97bbb35ad59c5817f27df65 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 28 Jan 2021 15:11:13 -0500
Subject: [PATCH 26/50] ... this?

---
 src/allmydata/test/eliotutil.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index 3579eb31f..f91d76e03 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -134,7 +134,7 @@ class EliotLoggedRunTest(object):
 
 
 def with_logging(
-        test_id,      # type: unicode
+        test_id,      # type: str
         test_method,  # type: Callable
 ):
     """

From be59682532b2243054423541461a8725c84d6f4b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 10:43:01 -0500
Subject: [PATCH 27/50] Fix issue with file uploads being Unicode on Python 3.

---
 src/allmydata/webish.py | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index c3965645a..2d9ef25ff 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -44,6 +44,42 @@ from .web.storage_plugins import (
     StoragePlugins,
 )
 
+
+if PY2:
+    FileUploadFieldStorage = FieldStorage
+else:
+    class FileUploadFieldStorage(FieldStorage):
+        """
+        Do terrible things to ensure files are still bytes.
+
+        On Python 2, uploaded files were always bytes.  On Python 3, there's a
+        heuristic: if the filename is set on a field, it's assumed to be a file
+        upload and therefore bytes.  If no filename is set, it's Unicode.
+
+        Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
+        enables setting the filename not via the MIME filename, but via a
+        separate field called "name".
+
+        Thus we need to do this ridiculous workaround.
+
+        Source for idea:
+        https://mail.python.org/pipermail/python-dev/2017-February/147402.html
+        """
+        @property
+        def filename(self):
+            if self.name == "file" and not self._mime_filename:
+                # We use the file field to upload files, see directory.py's
+                # _POST_upload. Lack of _mime_filename means we need to trick
+                # FieldStorage into thinking there is a filename so it'll
+                # return bytes.
+                return "unknown-filename"
+            return self._mime_filename
+
+        @filename.setter
+        def filename(self, value):
+            self._mime_filename = value
+
+
 class TahoeLAFSRequest(Request, object):
     """
     ``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
@@ -94,7 +130,8 @@ class TahoeLAFSRequest(Request, object):
                 headers['content-length'] = str(self.content.tell())
                 self.content.seek(0)
 
-            self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'})
+            self.fields = FileUploadFieldStorage(
+                self.content, headers, environ={'REQUEST_METHOD': 'POST'})
             self.content.seek(0)
 
         self._tahoeLAFSSecurityPolicy()

From 20c04f4ddb2b9236dc206ceadc1bcc51909c67bc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 10:46:08 -0500
Subject: [PATCH 28/50] Make sure it's non-UTF-8-encodable bytes, just to make
 sure it's roundtripping correctly.

---
 src/allmydata/test/web/test_web.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 4ef9405e5..b0db81839 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -3094,13 +3094,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                               self.NEWFILE_CONTENTS)
 
     def test_POST_upload_named(self):
+        NEWFILE_CONTENTS = self.NEWFILE_CONTENTS + b"\xFF\x00\xFF"
         fn = self._foo_node
         d = self.POST(self.public_url + "/foo", t="upload",
-                      name="new.txt", file=self.NEWFILE_CONTENTS)
+                      name="new.txt", file=NEWFILE_CONTENTS)
         d.addCallback(self.failUnlessURIMatchesROChild, fn, u"new.txt")
         d.addCallback(lambda res:
                       self.failUnlessChildContentsAre(fn, u"new.txt",
-                                                      self.NEWFILE_CONTENTS))
+                                                      NEWFILE_CONTENTS))
         return d
 
     def test_POST_upload_named_badfilename(self):

From 438933ec8d8ed1d688456c574113ef5a23cc0037 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 11:02:00 -0500
Subject: [PATCH 29/50] Fix tests on Python 2.

---
 src/allmydata/test/web/test_web.py | 38 ++++++++++++++++--------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index b0db81839..de8b07a94 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -568,35 +568,37 @@ class WebMixin(TimezoneMixin):
         return do_http("delete", url)
 
     def build_form(self, **fields):
-        sepbase = "boogabooga"
-        sep = "--" + sepbase
+        sepbase = b"boogabooga"
+        sep = b"--" + sepbase
         form = []
         form.append(sep)
-        form.append('Content-Disposition: form-data; name="_charset"')
-        form.append('')
-        form.append('UTF-8')
+        form.append(b'Content-Disposition: form-data; name="_charset"')
+        form.append(b'')
+        form.append(b'UTF-8')
         form.append(sep)
         for name, value in fields.items():
+            if isinstance(name, unicode):
+                name = name.encode("utf-8")
             if isinstance(value, tuple):
                 filename, value = value
-                form.append('Content-Disposition: form-data; name="%s"; '
-                            'filename="%s"' % (name, filename))
+                if isinstance(filename, unicode):
+                    filename = filename.encode("utf-8")
+                form.append(b'Content-Disposition: form-data; name="%s"; '
+                            b'filename="%s"' % (name, filename))
             else:
-                form.append('Content-Disposition: form-data; name="%s"' % name)
-            form.append('')
-            if isinstance(value, bytes):
-                value = unicode(value, "utf-8")
-            if not isinstance(value, unicode):
-                value = unicode(value)
+                form.append(b'Content-Disposition: form-data; name="%s"' % name)
+            form.append(b'')
+            if isinstance(value, unicode):
+                value = value.encode("utf-8")
             form.append(value)
             form.append(sep)
-        form[-1] += "--"
-        body = ""
+        form[-1] += b"--"
+        body = b""
         headers = {}
         if fields:
-            body = "\r\n".join(form) + "\r\n"
-            headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
-        return (body.encode("utf-8"), headers)
+            body = b"\r\n".join(form) + b"\r\n"
+            headers["content-type"] = "multipart/form-data; boundary=%s" % unicode(sepbase, "utf-8")
+        return (body, headers)
 
     def POST(self, urlpath, **fields):
         body, headers = self.build_form(**fields)

From 73537351a7b5302659707c4861957a2468deb9a5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 11:05:36 -0500
Subject: [PATCH 30/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index de8b07a94..a02459007 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -2688,6 +2688,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessCHKURIHasContents(self, got_uri, contents):
+        if isinstance(got_uri, unicode):
+            got_uri = got_uri.encode("utf-8")
         self.failUnless(self.get_all_contents()[got_uri] == contents)
 
     def test_POST_upload(self):
@@ -2737,6 +2739,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.POST("/uri", t="upload",
                       file=("new.txt", self.NEWFILE_CONTENTS))
         def _check_upload_results(page):
+            page = unicode(page, "utf-8")
             # this should be a page which describes the results of the upload
             # that just finished.
             self.failUnlessIn("Upload Results:", page)
@@ -2778,7 +2781,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                       file=("new.txt", self.NEWFILE_CONTENTS))
         def _check(filecap):
             filecap = filecap.strip()
-            self.failUnless(filecap.startswith("URI:SSK:"), filecap)
+            self.failUnless(filecap.startswith(b"URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())

From a27dc83920c02183f83f4cfae15a2a096b2c8893 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 11:42:09 -0500
Subject: [PATCH 31/50] More passing tests on Python 3.

---
 src/allmydata/web/common.py   | 2 +-
 src/allmydata/web/unlinked.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index b642a0b28..0a39d7336 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -674,7 +674,7 @@ def url_for_string(req, url_string):
         and the given URL string.
     """
     url = DecodedURL.from_text(url_string.decode("utf-8"))
-    if url.host == b"":
+    if not url.host:
         root = req.URLPath()
         netloc = root.netloc.split(b":", 1)
         if len(netloc) == 1:
diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py
index 2e6495957..cc13ca1a7 100644
--- a/src/allmydata/web/unlinked.py
+++ b/src/allmydata/web/unlinked.py
@@ -66,8 +66,8 @@ def POSTUnlinkedCHK(req, client):
         # if when_done= is provided, return a redirect instead of our
         # usual upload-results page
         def _done(upload_results, redir_to):
-            if "%(uri)s" in redir_to:
-                redir_to = redir_to.replace("%(uri)s", urlquote(upload_results.get_uri()))
+            if b"%(uri)s" in redir_to:
+                redir_to = redir_to.replace(b"%(uri)s", urlquote(upload_results.get_uri()).encode("utf-8"))
             return url_for_string(req, redir_to)
         d.addCallback(_done, when_done)
     else:

From abf6413a8925b1cfbf3bf25c2f91dd6559a82085 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 1 Feb 2021 11:50:33 -0500
Subject: [PATCH 32/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index a02459007..00af471a5 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1500,7 +1500,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_FILE_URI_mdmf(self):
         base = "/uri/%s" % urlquote(unicode(self._quux_txt_uri, "ascii"))
-        self._quux_new_contents = "new_contents"
+        self._quux_new_contents = b"new_contents"
         d = self.GET(base)
         d.addCallback(lambda res:
             self.failUnlessIsQuuxDotTxt(res))
@@ -1533,7 +1533,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ignored:
             self.shouldFail2(error.Error, "test_PUT_FILE_URI_mdmf_readonly",
                              "400 Bad Request", "read-only cap",
-                             self.PUT, base, "new data"))
+                             self.PUT, base, b"new data"))
         return d
 
     def test_PUT_FILE_URI_sdmf_readonly(self):
@@ -1545,7 +1545,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ignored:
             self.shouldFail2(error.Error, "test_PUT_FILE_URI_sdmf_readonly",
                              "400 Bad Request", "read-only cap",
-                             self.PUT, base, "new_data"))
+                             self.PUT, base, b"new_data"))
         return d
 
     def test_GET_etags(self):
@@ -1732,11 +1732,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.PUT("/uri?format=mdmf",
                      contents)
         def _got_filecap(filecap):
-            self.failUnless(filecap.startswith("URI:MDMF"))
+            self.failUnless(filecap.startswith(b"URI:MDMF"))
             return filecap
         d.addCallback(_got_filecap)
         d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
-        d.addCallback(lambda json: self.failUnlessIn("MDMF", json))
+        d.addCallback(lambda json: self.failUnlessIn(b"MDMF", json))
         return d
 
     def test_PUT_NEWFILEURL_unlinked_sdmf(self):
@@ -1789,7 +1789,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # It is okay to upload large mutable files, so we should be able
         # to do that.
         d = self.PUT(self.public_url + "/foo/new.txt?mutable=true",
-                     "b" * (self.s.MUTABLE_SIZELIMIT + 1))
+                     b"b" * (self.s.MUTABLE_SIZELIMIT + 1))
         return d
 
     def test_PUT_NEWFILEURL_replace(self):
@@ -1804,9 +1804,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_NEWFILEURL_bad_t(self):
         d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
-                             "PUT to a file: bad t=bogus",
+                             "PUT to a file: bad t=",
                              self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
-                             "contents")
+                             b"contents")
         return d
 
     def test_PUT_NEWFILEURL_no_replace(self):
@@ -2806,7 +2806,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # able to upload mutable files that are as large as we want them
         # to be.
         d = self.POST("/uri", t="upload", mutable="true",
-                      file=("new.txt", "b" * (self.s.MUTABLE_SIZELIMIT + 1)))
+                      file=("new.txt", b"b" * (self.s.MUTABLE_SIZELIMIT + 1)))
         return d
 
 

From 26560a4212c326a12210459ae797b9d508e6fea9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 2 Feb 2021 14:25:01 -0500
Subject: [PATCH 33/50] More passing tests on Python 3.

---
 src/allmydata/test/web/test_web.py | 60 +++++++++++++++---------------
 src/allmydata/web/filenode.py      |  4 +-
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 00af471a5..9d73c9047 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1735,7 +1735,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(filecap.startswith(b"URI:MDMF"))
             return filecap
         d.addCallback(_got_filecap)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8")))
         d.addCallback(lambda json: self.failUnlessIn(b"MDMF", json))
         return d
 
@@ -1743,8 +1743,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         contents = self.NEWFILE_CONTENTS * 300000
         d = self.PUT("/uri?format=sdmf",
                      contents)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % filecap))
-        d.addCallback(lambda json: self.failUnlessIn("SDMF", json))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8")))
+        d.addCallback(lambda json: self.failUnlessIn(b"SDMF", json))
         return d
 
     @inlineCallbacks
@@ -4495,14 +4495,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILE_URI(self):
-        file_contents = "New file contents here\n"
+        file_contents = b"New file contents here\n"
         d = self.PUT("/uri", file_contents)
         def _check(uri):
-            assert isinstance(uri, str), uri
+            assert isinstance(uri, bytes), uri
             self.failUnlessIn(uri, self.get_all_contents())
             self.failUnlessReallyEqual(self.get_all_contents()[uri],
                                        file_contents)
-            return self.GET("/uri/%s" % uri)
+            return self.GET("/uri/%s" % unicode(uri, "utf-8"))
         d.addCallback(_check)
         def _check2(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4510,14 +4510,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILE_URI_not_mutable(self):
-        file_contents = "New file contents here\n"
+        file_contents = b"New file contents here\n"
         d = self.PUT("/uri?mutable=false", file_contents)
         def _check(uri):
-            assert isinstance(uri, str), uri
+            assert isinstance(uri, bytes), uri
             self.failUnlessIn(uri, self.get_all_contents())
             self.failUnlessReallyEqual(self.get_all_contents()[uri],
                                        file_contents)
-            return self.GET("/uri/%s" % uri)
+            return self.GET("/uri/%s" % unicode(uri, "utf-8"))
         d.addCallback(_check)
         def _check2(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4525,7 +4525,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILE_URI_only_PUT(self):
-        d = self.PUT("/uri?t=bogus", "")
+        d = self.PUT("/uri?t=bogus", b"")
         d.addBoth(self.shouldFail, error.Error,
                   "PUT_NEWFILE_URI_only_PUT",
                   "400 Bad Request",
@@ -4533,11 +4533,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_NEWFILE_URI_mutable(self):
-        file_contents = "New file contents here\n"
+        file_contents = b"New file contents here\n"
         d = self.PUT("/uri?mutable=true", file_contents)
         def _check1(filecap):
             filecap = filecap.strip()
-            self.failUnless(filecap.startswith("URI:SSK:"), filecap)
+            self.failUnless(filecap.startswith(b"URI:SSK:"), filecap)
             self.filecap = filecap
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
             self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
@@ -4546,7 +4546,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check1)
         def _check2(data):
             self.failUnlessReallyEqual(data, file_contents)
-            return self.GET("/uri/%s" % urlquote(self.filecap))
+            return self.GET("/uri/%s" % urlquote(unicode(self.filecap, "utf-8")))
         d.addCallback(_check2)
         def _check3(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4559,7 +4559,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             n = self.s.create_node_from_uri(uri.strip())
             d2 = self.failUnlessNodeKeysAre(n, [])
             d2.addCallback(lambda res:
-                           self.GET("/uri/%s?t=json" % uri))
+                           self.GET("/uri/%s?t=json" % unicode(uri, "utf-8")))
             return d2
         d.addCallback(_check)
         d.addCallback(self.failUnlessIsEmptyJSON)
@@ -4586,7 +4586,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_mkdir_bad_format(self):
         url = self.webish_url + "/uri?t=mkdir&format=foo"
         yield self.assertHTTPError(url, 400, "Unknown format:",
-                                   method="put", data="")
+                                   method="put", data=b"")
 
     def test_POST_check(self):
         d = self.POST(self.public_url + "/foo", t="check", name="bar.txt")
@@ -4602,20 +4602,20 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
 
     def test_PUT_update_at_offset(self):
-        file_contents = "test file" * 100000 # about 900 KiB
+        file_contents = b"test file" * 100000 # about 900 KiB
         d = self.PUT("/uri?mutable=true", file_contents)
         def _then(filecap):
             self.filecap = filecap
             new_data = file_contents[:100]
-            new = "replaced and so on"
+            new = b"replaced and so on"
             new_data += new
             new_data += file_contents[len(new_data):]
             assert len(new_data) == len(file_contents)
             self.new_data = new_data
         d.addCallback(_then)
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?replace=True&offset=100" % self.filecap,
-                     "replaced and so on"))
+            self.PUT("/uri/%s?replace=True&offset=100" % unicode(self.filecap, "utf-8"),
+                     b"replaced and so on"))
         def _get_data(filecap):
             n = self.s.create_node_from_uri(filecap)
             return n.download_best_version()
@@ -4624,35 +4624,35 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessEqual(results, self.new_data))
         # Now try appending things to the file
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?offset=%d" % (self.filecap, len(self.new_data)),
-                     "puppies" * 100))
+            self.PUT("/uri/%s?offset=%d" % (unicode(self.filecap, "utf-8"), len(self.new_data)),
+                     b"puppies" * 100))
         d.addCallback(_get_data)
         d.addCallback(lambda results:
-            self.failUnlessEqual(results, self.new_data + ("puppies" * 100)))
+            self.failUnlessEqual(results, self.new_data + (b"puppies" * 100)))
         # and try replacing the beginning of the file
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?offset=0" % self.filecap, "begin"))
+            self.PUT("/uri/%s?offset=0" % unicode(self.filecap, "utf-8"), b"begin"))
         d.addCallback(_get_data)
         d.addCallback(lambda results:
-            self.failUnlessEqual(results, "begin"+self.new_data[len("begin"):]+("puppies"*100)))
+            self.failUnlessEqual(results, b"begin"+self.new_data[len(b"begin"):]+(b"puppies"*100)))
         return d
 
     @inlineCallbacks
     def test_PUT_update_at_invalid_offset(self):
-        file_contents = "test file" * 100000 # about 900 KiB
+        file_contents = b"test file" * 100000 # about 900 KiB
         filecap = yield self.PUT("/uri?mutable=true", file_contents)
         # Negative offsets should cause an error.
-        url = self.webish_url + "/uri/%s?offset=-1" % filecap
+        url = self.webish_url + "/uri/%s?offset=-1" % unicode(filecap, "utf-8")
         yield self.assertHTTPError(url, 400, "Invalid offset",
-                                   method="put", data="foo")
+                                   method="put", data=b"foo")
 
     @inlineCallbacks
     def test_PUT_update_at_offset_immutable(self):
-        file_contents = "Test file" * 100000
+        file_contents = b"Test file" * 100000
         filecap = yield self.PUT("/uri", file_contents)
-        url = self.webish_url + "/uri/%s?offset=50" % filecap
+        url = self.webish_url + "/uri/%s?offset=50" % unicode(filecap, "utf-8")
         yield self.assertHTTPError(url, 400, "immutable",
-                                   method="put", data="foo")
+                                   method="put", data=b"foo")
 
     @inlineCallbacks
     def test_bad_method(self):
diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py
index d050fb8a0..4345d4b4a 100644
--- a/src/allmydata/web/filenode.py
+++ b/src/allmydata/web/filenode.py
@@ -127,7 +127,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
                            http.NOT_IMPLEMENTED)
         if not t:
             return self.replace_me_with_a_child(req, self.client, replace)
-        if t == "uri":
+        if t == b"uri":
             return self.replace_me_with_a_childcap(req, self.client, replace)
 
         raise WebError("PUT to a file: bad t=%s" % t)
@@ -281,7 +281,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
                 assert self.parentnode and self.name
                 return self.replace_me_with_a_child(req, self.client, replace)
 
-        if t == "uri":
+        if t == b"uri":
             if not replace:
                 raise ExistingChildError()
             assert self.parentnode and self.name

From 489b9f87b3fe056880e65fdd520135deddfe53c7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 2 Feb 2021 15:14:44 -0500
Subject: [PATCH 34/50] All tests pass on Python 3.

---
 src/allmydata/test/web/test_web.py | 42 ++++++++++++++++--------------
 src/allmydata/web/operations.py    |  2 +-
 src/allmydata/web/root.py          |  6 ++---
 src/allmydata/web/status.py        | 14 +++++-----
 4 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 9d73c9047..f8eabaed9 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -832,7 +832,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                      u'nickname': u'disconnected_nickname \u263b',
                      u'version': u'1.0',
                     },
-                ]),
+                ], key=lambda o: sorted(o.items())),
             }
             self.assertEqual(expected, decoded)
         d.addCallback(_check)
@@ -855,7 +855,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_set_introducer_not_connected_unguessable)
         def _check_introducer_not_connected_unguessable(res):
             soup = BeautifulSoup(res, 'html5lib')
-            self.failIfIn('pb://someIntroducer/secret', res)
+            self.failIfIn(b'pb://someIntroducer/secret', res)
             assert_soup_has_tag_with_attributes(
                 self, soup, u"img",
                 {u"alt": u"Disconnected", u"src": u"img/connected-no.png"}
@@ -879,7 +879,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 u"summary",
                 { u"class": u"connection-status", u"title": u"(no other hints)" }
             )
-            self.failIfIn('pb://someIntroducer/secret', res)
+            self.failIfIn(b'pb://someIntroducer/secret', res)
             assert_soup_has_tag_with_attributes(
                 self, soup, u"img",
                 { u"alt": u"Connected", u"src": u"img/connected-yes.png" }
@@ -943,7 +943,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 u"pb://someHelper/[censored]",
                 { u"class": u"furl" }
             )
-            self.failIfIn('pb://someHelper/secret', res)
+            self.failIfIn(b'pb://someHelper/secret', res)
             assert_soup_has_tag_with_attributes(
                 self, soup, u"img",
                 { u"src": u"img/connected-no.png", u"alt": u"Disconnected" }
@@ -963,7 +963,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 u"pb://someHelper/[censored]",
                 { u"class": u"furl" }
             )
-            self.failIfIn('pb://someHelper/secret', res)
+            self.failIfIn(b'pb://someHelper/secret', res)
             assert_soup_has_tag_with_attributes(
                 self, soup, u"img",
                 { u"src": u"img/connected-yes.png", "alt": u"Connected" }
@@ -991,6 +991,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
         d = self.GET("/status", followRedirect=True)
         def _check(res):
+            res = unicode(res, "utf-8")
             self.failUnlessIn('Recent and Active Operations', res)
             self.failUnlessIn('"/status/down-%d"' % dl_num, res)
             self.failUnlessIn('"/status/up-%d"' % ul_num, res)
@@ -1009,7 +1010,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         d.addCallback(lambda res: self.GET("/status/down-%d" % dl_num))
         def _check_dl(res):
-            self.failUnlessIn("File Download Status", res)
+            self.failUnlessIn(b"File Download Status", res)
         d.addCallback(_check_dl)
         d.addCallback(lambda res: self.GET("/status/down-%d/event_json" % dl_num))
         def _check_dl_json(res):
@@ -1020,12 +1021,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
-            phwr_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_a")[:20])
-            cmpu_id = base32.b2a(hashutil.tagged_hash("foo", "serverid_b")[:20])
+            phwr_id = unicode(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_a")[:20]), "ascii")
+            cmpu_id = unicode(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_b")[:20]), "ascii")
             # serverids[] keys are strings, since that's what JSON does, but
             # we'd really like them to be ints
             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
-            self.failUnless(data["serverids"].has_key("1"),
+            self.failUnless("1" in data["serverids"],
                             str(data["serverids"]))
             self.failUnlessEqual(data["serverids"]["1"], "cmpuvkjm",
                                  str(data["serverids"]))
@@ -1038,19 +1039,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check_dl_json)
         d.addCallback(lambda res: self.GET("/status/up-%d" % ul_num))
         def _check_ul(res):
-            self.failUnlessIn("File Upload Status", res)
+            self.failUnlessIn(b"File Upload Status", res)
         d.addCallback(_check_ul)
         d.addCallback(lambda res: self.GET("/status/mapupdate-%d" % mu_num))
         def _check_mapupdate(res):
-            self.failUnlessIn("Mutable File Servermap Update Status", res)
+            self.failUnlessIn(b"Mutable File Servermap Update Status", res)
         d.addCallback(_check_mapupdate)
         d.addCallback(lambda res: self.GET("/status/publish-%d" % pub_num))
         def _check_publish(res):
-            self.failUnlessIn("Mutable File Publish Status", res)
+            self.failUnlessIn(b"Mutable File Publish Status", res)
         d.addCallback(_check_publish)
         d.addCallback(lambda res: self.GET("/status/retrieve-%d" % ret_num))
         def _check_retrieve(res):
-            self.failUnlessIn("Mutable File Retrieve Status", res)
+            self.failUnlessIn(b"Mutable File Retrieve Status", res)
         d.addCallback(_check_retrieve)
 
         return d
@@ -3700,6 +3701,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_welcome_page_mkdir_button(self):
         # Fetch the welcome page.
         res = yield self.GET("/")
+        res = unicode(res, "utf-8")
         MKDIR_BUTTON_RE = re.compile(
             '<form(?: action="([^"]*)"| method="post"| enctype="multipart/form-data"){3}>.*'
             '<input (?:type="hidden" |name="t" |value="([^"]*?)" ){3}/>[ ]*'
@@ -3718,7 +3720,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         target = yield self.shouldRedirectTo(url, None,
                                              method="post",
                                              code=http.SEE_OTHER)
-        target = urlunquote(target)
+        target = urlunquote(unicode(target, "utf-8"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def test_POST_mkdir_replace(self): # return value?
@@ -4682,7 +4684,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         res = yield self.GET("/operations/128?t=status&output=JSON")
         data = json.loads(res)
         self.failUnless("finished" in data, res)
-        monitor = self.ws.getServiceNamed("operations").handles["128"][0]
+        monitor = self.ws.getServiceNamed("operations").handles[b"128"][0]
 
         res = yield self.POST("/operations/128?t=cancel&output=JSON")
         data = json.loads(res)
@@ -4801,7 +4803,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             dircap_uri,
         )
         self.assertEqual(
-            response.request.absoluteURI,
+            unicode(response.request.absoluteURI, "utf-8"),
             self.webish_url + "/uri/{}?t=json".format(urlquote(dircap)))
         if response.code >= 400:
             raise Error(response.code, response=response.content())
@@ -4809,8 +4811,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_incident(self):
         d = self.POST("/report_incident", details="eek")
         def _done(res):
-            self.failIfIn("<html>", res)
-            self.failUnlessIn("An incident report has been saved", res)
+            self.failIfIn(b"<html>", res)
+            self.failUnlessIn(b"An incident report has been saved", res)
         d.addCallback(_done)
         return d
 
@@ -4818,12 +4820,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         webdir = os.path.join(self.staticdir, "subdir")
         fileutil.make_dirs(webdir)
         f = open(os.path.join(webdir, "hello.txt"), "wb")
-        f.write("hello")
+        f.write(b"hello")
         f.close()
 
         d = self.GET("/static/subdir/hello.txt")
         def _check(res):
-            self.failUnlessReallyEqual(res, "hello")
+            self.failUnlessReallyEqual(res, b"hello")
         d.addCallback(_check)
         return d
 
diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py
index 6769fb044..8f78d8a0b 100644
--- a/src/allmydata/web/operations.py
+++ b/src/allmydata/web/operations.py
@@ -102,7 +102,7 @@ class OphandleTable(resource.Resource, service.Service):
     def getChild(self, name, req):
         ophandle = name
         if ophandle not in self.handles:
-            raise WebError("unknown/expired handle '%s'" % escape(ophandle),
+            raise WebError("unknown/expired handle '%s'" % escape(unicode(ophandle, "utf-8")),
                            NOT_FOUND)
         (monitor, renderer, when_added) = self.handles[ophandle]
 
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index c5474a6cb..49ad222d8 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -256,11 +256,11 @@ class Root(MultiFormatResource):
         if not path:
             # Render "/" path.
             return self
-        if path == "helper_status":
+        if path == b"helper_status":
             # the Helper isn't attached until after the Tub starts, so this child
             # needs to created on each request
             return status.HelperStatus(self._client.helper)
-        if path == "storage":
+        if path == b"storage":
             # Storage isn't initialized until after the web hierarchy is
             # constructed so this child needs to be created later than
             # `__init__`.
@@ -294,7 +294,7 @@ class Root(MultiFormatResource):
             self._describe_server(server)
             for server
             in broker.get_known_servers()
-        ))
+        ), key=lambda o: sorted(o.items()))
 
 
     def _describe_server(self, server):
diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py
index 2002b2fdf..3a717567e 100644
--- a/src/allmydata/web/status.py
+++ b/src/allmydata/web/status.py
@@ -284,7 +284,7 @@ def _find_overlap(events, start_key, end_key):
     rows = []
     for ev in events:
         ev = ev.copy()
-        if ev.has_key('server'):
+        if 'server' in ev:
             ev["serverid"] = ev["server"].get_longname()
             del ev["server"]
         # find an empty slot in the rows
@@ -362,8 +362,8 @@ def _find_overlap_requests(events):
 def _color(server):
     h = hashlib.sha256(server.get_serverid()).digest()
     def m(c):
-        return min(ord(c) / 2 + 0x80, 0xff)
-    return "#%02x%02x%02x" % (m(h[0]), m(h[1]), m(h[2]))
+        return min(ord(c) // 2 + 0x80, 0xff)
+    return "#%02x%02x%02x" % (m(h[0:1]), m(h[1:2]), m(h[2:3]))
 
 class _EventJson(Resource, object):
 
@@ -426,7 +426,7 @@ class DownloadStatusPage(Resource, object):
         """
         super(DownloadStatusPage, self).__init__()
         self._download_status = download_status
-        self.putChild("event_json", _EventJson(self._download_status))
+        self.putChild(b"event_json", _EventJson(self._download_status))
 
     @render_exception
     def render_GET(self, req):
@@ -1288,14 +1288,14 @@ class Status(MultiFormatResource):
         # final URL segment will be an empty string. Resources can
         # thus know if they were requested with or without a final
         # slash."
-        if not path and request.postpath != ['']:
+        if not path and request.postpath != [b'']:
             return self
 
         h = self.history
         try:
-            stype, count_s = path.split("-")
+            stype, count_s = path.split(b"-")
         except ValueError:
-            raise WebError("no '-' in '{}'".format(path))
+            raise WebError("no '-' in '{}'".format(unicode(path, "utf-8")))
         count = int(count_s)
         stype = unicode(stype, "ascii")
         if stype == "up":

From fa3bef5b258b52f6a299e549426c3e0e4cc3b978 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 2 Feb 2021 15:27:15 -0500
Subject: [PATCH 35/50] Port to Python 3.

---
 src/allmydata/test/common_util.py  |   5 +
 src/allmydata/test/web/test_web.py | 228 +++++++++++++++--------------
 src/allmydata/util/_python3.py     |   1 +
 3 files changed, 124 insertions(+), 110 deletions(-)

diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 2a70cff3a..0bff5e171 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -203,6 +203,11 @@ def flip_one_bit(s, offset=0, size=None):
 class ReallyEqualMixin(object):
     def failUnlessReallyEqual(self, a, b, msg=None):
         self.assertEqual(a, b, msg)
+        # Make sure unicode strings are a consistent type:
+        if a.__class__ == future_str:
+            a = unicode(a)
+        if b.__class__ == future_str:
+            b = unicode(b)
         self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
 
 
diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index f8eabaed9..4dbdf685b 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -1,6 +1,14 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
 
-from past.builtins import unicode
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from six import ensure_binary
 
 import os.path, re, time
@@ -343,7 +351,7 @@ class WebMixin(TimezoneMixin):
         def _then(res):
             self.public_root = res[0][1]
             assert interfaces.IDirectoryNode.providedBy(self.public_root), res
-            self.public_url = "/uri/" + unicode(self.public_root.get_uri(), "ascii")
+            self.public_url = "/uri/" + str(self.public_root.get_uri(), "ascii")
             self.private_root = res[1][1]
 
             foo = res[2][1]
@@ -481,9 +489,9 @@ class WebMixin(TimezoneMixin):
             self.failIfIn("rw_uri", metadata)
         else:
             self.failUnlessIn("rw_uri", metadata)
-            self.failUnlessEqual(metadata['rw_uri'], unicode(self._quux_txt_uri, "ascii"))
+            self.failUnlessEqual(metadata['rw_uri'], str(self._quux_txt_uri, "ascii"))
         self.failUnlessIn("ro_uri", metadata)
-        self.failUnlessEqual(metadata['ro_uri'], unicode(self._quux_txt_readonly_uri, "ascii"))
+        self.failUnlessEqual(metadata['ro_uri'], str(self._quux_txt_readonly_uri, "ascii"))
         self.failUnlessReallyEqual(metadata['size'], len(self.QUUX_CONTENTS))
 
     def failUnlessIsFooJSON(self, res):
@@ -497,13 +505,13 @@ class WebMixin(TimezoneMixin):
         self.failUnlessReallyEqual(to_bytes(data[1]["ro_uri"]), self._foo_readonly_uri)
         self.failUnlessReallyEqual(to_bytes(data[1]["verify_uri"]), self._foo_verifycap)
 
-        kidnames = sorted([unicode(n) for n in data[1]["children"]])
+        kidnames = sorted([str(n) for n in data[1]["children"]])
         self.failUnlessEqual(kidnames,
                              [self._htmlname_unicode, u"bar.txt", u"baz.txt",
                               u"blockingfile", u"empty", u"n\u00fc.txt", u"quux.txt", u"sub"])
-        kids = dict( [(unicode(name),value)
+        kids = dict( [(str(name),value)
                       for (name,value)
-                      in data[1]["children"].items()] )
+                      in list(data[1]["children"].items())] )
         self.failUnlessEqual(kids[u"sub"][0], "dirnode")
         self.failUnlessIn("metadata", kids[u"sub"][1])
         self.failUnlessIn("tahoe", kids[u"sub"][1]["metadata"])
@@ -576,19 +584,19 @@ class WebMixin(TimezoneMixin):
         form.append(b'')
         form.append(b'UTF-8')
         form.append(sep)
-        for name, value in fields.items():
-            if isinstance(name, unicode):
+        for name, value in list(fields.items()):
+            if isinstance(name, str):
                 name = name.encode("utf-8")
             if isinstance(value, tuple):
                 filename, value = value
-                if isinstance(filename, unicode):
+                if isinstance(filename, str):
                     filename = filename.encode("utf-8")
                 form.append(b'Content-Disposition: form-data; name="%s"; '
                             b'filename="%s"' % (name, filename))
             else:
                 form.append(b'Content-Disposition: form-data; name="%s"' % name)
             form.append(b'')
-            if isinstance(value, unicode):
+            if isinstance(value, str):
                 value = value.encode("utf-8")
             form.append(value)
             form.append(sep)
@@ -597,7 +605,7 @@ class WebMixin(TimezoneMixin):
         headers = {}
         if fields:
             body = b"\r\n".join(form) + b"\r\n"
-            headers["content-type"] = "multipart/form-data; boundary=%s" % unicode(sepbase, "utf-8")
+            headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "utf-8")
         return (body, headers)
 
     def POST(self, urlpath, **fields):
@@ -606,7 +614,7 @@ class WebMixin(TimezoneMixin):
 
     def POST2(self, urlpath, body="", headers={}, followRedirect=False):
         url = self.webish_url + urlpath
-        if isinstance(body, unicode):
+        if isinstance(body, str):
             body = body.encode("utf-8")
         return do_http("POST", url, allow_redirects=followRedirect,
                        headers=headers, data=body)
@@ -641,7 +649,7 @@ class WebMixin(TimezoneMixin):
                 if response_substring:
                     response = res.value.response
                     if isinstance(response, bytes):
-                        response = unicode(response, "utf-8")
+                        response = str(response, "utf-8")
                     self.failUnlessIn(response_substring, response,
                                       "'%s' not in '%s' for test '%s'" % \
                                       (response_substring, res.value.response,
@@ -675,7 +683,7 @@ class MultiFormatResourceTests(TrialTestCase):
     """
     def render(self, resource, **queryargs):
         # Query arguments in real twisted.web requests have byte keys.
-        queryargs = {k.encode("utf-8"): v for (k, v) in queryargs.items()}
+        queryargs = {k.encode("utf-8"): v for (k, v) in list(queryargs.items())}
         return self.successResultOf(render(resource, queryargs))
 
     def resource(self):
@@ -991,7 +999,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         ret_num = h.list_all_retrieve_statuses()[0].get_counter()
         d = self.GET("/status", followRedirect=True)
         def _check(res):
-            res = unicode(res, "utf-8")
+            res = str(res, "utf-8")
             self.failUnlessIn('Recent and Active Operations', res)
             self.failUnlessIn('"/status/down-%d"' % dl_num, res)
             self.failUnlessIn('"/status/up-%d"' % ul_num, res)
@@ -1021,8 +1029,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessEqual(data["segment"][0]["segment_length"], 100)
             self.failUnlessEqual(data["segment"][2]["segment_number"], 2)
             self.failUnlessEqual(data["segment"][2]["finish_time"], None)
-            phwr_id = unicode(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_a")[:20]), "ascii")
-            cmpu_id = unicode(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_b")[:20]), "ascii")
+            phwr_id = str(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_a")[:20]), "ascii")
+            cmpu_id = str(base32.b2a(hashutil.tagged_hash(b"foo", b"serverid_b")[:20]), "ascii")
             # serverids[] keys are strings, since that's what JSON does, but
             # we'd really like them to be ints
             self.failUnlessEqual(data["serverids"]["0"], "phwrsjte")
@@ -1472,19 +1480,19 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urlquote("%s:RANDOMSTUFF" % unicode(self._quux_txt_uri, "ascii"))
+        base = "/uri/%s" % urlquote("%s:RANDOMSTUFF" % str(self._quux_txt_uri, "ascii"))
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_mdmf_readonly(self):
-        base = "/uri/%s" % urlquote(unicode(self._quux_txt_readonly_uri, "ascii"))
+        base = "/uri/%s" % urlquote(str(self._quux_txt_readonly_uri, "ascii"))
         d = self.GET(base)
         d.addCallback(self.failUnlessIsQuuxDotTxt)
         return d
 
     def test_GET_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urlquote(unicode(self._bar_txt_uri, "ascii"))
+        base = "/uri/%s/boguschild" % urlquote(str(self._bar_txt_uri, "ascii"))
         errmsg = "Files have no children named 'boguschild'"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "400 Bad Request", errmsg,
@@ -1492,7 +1500,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_badchild(self):
-        base = "/uri/%s/boguschild" % urlquote(unicode(self._bar_txt_uri, "ascii"))
+        base = "/uri/%s/boguschild" % urlquote(str(self._bar_txt_uri, "ascii"))
         errmsg = "Cannot create directory 'boguschild', because its parent is a file, not a directory"
         d = self.shouldFail2(error.Error, "test_GET_FILE_URI_badchild",
                              "409 Conflict", errmsg,
@@ -1500,7 +1508,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf(self):
-        base = "/uri/%s" % urlquote(unicode(self._quux_txt_uri, "ascii"))
+        base = "/uri/%s" % urlquote(str(self._quux_txt_uri, "ascii"))
         self._quux_new_contents = b"new_contents"
         d = self.GET(base)
         d.addCallback(lambda res:
@@ -1514,7 +1522,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_PUT_FILE_URI_mdmf_extensions(self):
-        base = "/uri/%s" % urlquote("%s:EXTENSIONSTUFF" % unicode(self._quux_txt_uri, "ascii"))
+        base = "/uri/%s" % urlquote("%s:EXTENSIONSTUFF" % str(self._quux_txt_uri, "ascii"))
         self._quux_new_contents = b"new_contents"
         d = self.GET(base)
         d.addCallback(lambda res: self.failUnlessIsQuuxDotTxt(res))
@@ -1526,7 +1534,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_FILE_URI_mdmf_readonly(self):
         # We're not allowed to PUT things to a readonly cap.
-        base = "/uri/%s" % unicode(self._quux_txt_readonly_uri, "ascii")
+        base = "/uri/%s" % str(self._quux_txt_readonly_uri, "ascii")
         d = self.GET(base)
         d.addCallback(lambda res:
             self.failUnlessIsQuuxDotTxt(res))
@@ -1539,7 +1547,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def test_PUT_FILE_URI_sdmf_readonly(self):
         # We're not allowed to put things to a readonly cap.
-        base = "/uri/%s" % unicode(self._baz_txt_readonly_uri, "ascii")
+        base = "/uri/%s" % str(self._baz_txt_readonly_uri, "ascii")
         d = self.GET(base)
         d.addCallback(lambda res:
             self.failUnlessIsBazDotTxt(res))
@@ -1590,7 +1598,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         # use the ETag on GET
         def _check_match(ign):
-            uri = "/uri/%s" % unicode(self._bar_txt_uri, "ascii")
+            uri = "/uri/%s" % str(self._bar_txt_uri, "ascii")
             d = self.GET(uri, return_response=True)
             # extract the ETag
             d.addCallback(lambda data_code_headers:
@@ -1606,13 +1614,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check_match)
 
         def _no_etag(uri, t):
-            target = "/uri/%s?t=%s" % (unicode(uri, "ascii"), t)
+            target = "/uri/%s?t=%s" % (str(uri, "ascii"), t)
             d = self.GET(target, return_response=True, followRedirect=True)
             d.addCallback(lambda data_code_headers:
                           self.failIf(data_code_headers[2].hasHeader("etag"), target))
             return d
         def _yes_etag(uri, t):
-            target = "/uri/%s?t=%s" % (unicode(uri, "ascii"), t)
+            target = "/uri/%s?t=%s" % (str(uri, "ascii"), t)
             d = self.GET(target, return_response=True, followRedirect=True)
             d.addCallback(lambda data_code_headers:
                           self.failUnless(data_code_headers[2].hasHeader("etag"), target))
@@ -1651,7 +1659,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_info_mdmf(self):
-        d = self.GET("/uri/%s?t=info" % unicode(self._quux_txt_uri, "ascii"))
+        d = self.GET("/uri/%s?t=info" % str(self._quux_txt_uri, "ascii"))
         def _got(res):
             self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failUnlessIn(self._quux_txt_uri, res)
@@ -1660,7 +1668,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_info_mdmf_readonly(self):
-        d = self.GET("/uri/%s?t=info" % unicode(self._quux_txt_readonly_uri, "ascii"))
+        d = self.GET("/uri/%s?t=info" % str(self._quux_txt_readonly_uri, "ascii"))
         def _got(res):
             self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failIfIn(self._quux_txt_uri, res)
@@ -1669,7 +1677,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_info_sdmf(self):
-        d = self.GET("/uri/%s?t=info" % unicode(self._baz_txt_uri, "ascii"))
+        d = self.GET("/uri/%s?t=info" % str(self._baz_txt_uri, "ascii"))
         def _got(res):
             self.failUnlessIn(b"mutable file (sdmf)", res)
             self.failUnlessIn(self._baz_txt_uri, res)
@@ -1677,7 +1685,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_FILEURL_info_mdmf_extensions(self):
-        d = self.GET("/uri/%s:STUFF?t=info" % unicode(self._quux_txt_uri, "ascii"))
+        d = self.GET("/uri/%s:STUFF?t=info" % str(self._quux_txt_uri, "ascii"))
         def _got(res):
             self.failUnlessIn(b"mutable file (mdmf)", res)
             self.failUnlessIn(self._quux_txt_uri, res)
@@ -1736,7 +1744,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnless(filecap.startswith(b"URI:MDMF"))
             return filecap
         d.addCallback(_got_filecap)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8")))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % str(filecap, "utf-8")))
         d.addCallback(lambda json: self.failUnlessIn(b"MDMF", json))
         return d
 
@@ -1744,7 +1752,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         contents = self.NEWFILE_CONTENTS * 300000
         d = self.PUT("/uri?format=sdmf",
                      contents)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8")))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % str(filecap, "utf-8")))
         d.addCallback(lambda json: self.failUnlessIn(b"SDMF", json))
         return d
 
@@ -1890,7 +1898,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         # file is SDMF or MDMF
         d = self.PUT("/uri?format=mdmf",
                      self.NEWFILE_CONTENTS * 300000)
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "ascii")))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % str(filecap, "ascii")))
         def _got_json(raw, version):
             data = json.loads(raw)
             assert "filenode" == data[0]
@@ -1905,12 +1913,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda ignored:
             self.PUT("/uri?format=sdmf",
                       self.NEWFILE_CONTENTS * 300000))
-        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % unicode(filecap, "ascii")))
+        d.addCallback(lambda filecap: self.GET("/uri/%s?t=json" % str(filecap, "ascii")))
         d.addCallback(_got_json, "SDMF")
         return d
 
     def test_GET_FILEURL_json_mdmf(self):
-        d = self.GET("/uri/%s?t=json" % urlquote(unicode(self._quux_txt_uri, "ascii")))
+        d = self.GET("/uri/%s?t=json" % urlquote(str(self._quux_txt_uri, "ascii")))
         d.addCallback(self.failUnlessIsQuuxJSON)
         return d
 
@@ -2034,7 +2042,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                 for td in soup.find_all(u"td", {u"align": u"right"})
             )
         )
-        foo_url = urlquote("{}/uri/{}/".format(root, unicode(self._foo_uri, "ascii")))
+        foo_url = urlquote("{}/uri/{}/".format(root, str(self._foo_uri, "ascii")))
         forms = soup.find_all(u"form", {u"action": foo_url})
         found = []
         for form in forms:
@@ -2169,7 +2177,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             soup = BeautifulSoup(manifest, 'html5lib')
             assert_soup_has_text(self, soup, "Manifest of SI=")
             assert_soup_has_text(self, soup, "sub")
-            assert_soup_has_text(self, soup, unicode(self._sub_uri, "ascii"))
+            assert_soup_has_text(self, soup, str(self._sub_uri, "ascii"))
             assert_soup_has_text(self, soup, "sub/baz.txt")
             assert_soup_has_favicon(self, soup)
         d.addCallback(_got_html)
@@ -2261,7 +2269,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                         "largest-immutable-file": 19,
                         "api-version": 1,
                         }
-            for k,v in expected.items():
+            for k,v in list(expected.items()):
                 self.failUnlessReallyEqual(stats[k], v,
                                            "stats[%s] was %s, not %s" %
                                            (k, stats[k], v))
@@ -2393,7 +2401,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                        json.dumps(newkids))
         def _check(uri):
             n = self.s.create_node_from_uri(uri.strip())
-            d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
+            d2 = self.failUnlessNodeKeysAre(n, list(newkids.keys()))
             self.failUnlessEqual(n._node.get_version(), version)
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
@@ -2427,7 +2435,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
-        d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
+        d.addCallback(self.failUnlessNodeKeysAre, list(newkids.keys()))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
@@ -2455,7 +2463,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                        json.dumps(newkids))
         def _check(uri):
             n = self.s.create_node_from_uri(uri.strip())
-            d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
+            d2 = self.failUnlessNodeKeysAre(n, list(newkids.keys()))
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                        caps['filecap1']))
@@ -2476,7 +2484,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
-        d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
+        d.addCallback(self.failUnlessNodeKeysAre, list(newkids.keys()))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
@@ -2601,21 +2609,21 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
     def failUnlessNodeKeysAre(self, node, expected_keys):
         for k in expected_keys:
-            assert isinstance(k, unicode)
+            assert isinstance(k, str)
         d = node.list()
         def _check(children):
             self.failUnlessReallyEqual(sorted(children.keys()), sorted(expected_keys))
         d.addCallback(_check)
         return d
     def failUnlessNodeHasChild(self, node, name):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.list()
         def _check(children):
             self.failUnlessIn(name, children)
         d.addCallback(_check)
         return d
     def failIfNodeHasChild(self, node, name):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.list()
         def _check(children):
             self.failIfIn(name, children)
@@ -2623,7 +2631,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessChildContentsAre(self, node, name, expected_contents):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         d.addCallback(lambda node: download_to_data(node))
         def _check(contents):
@@ -2632,7 +2640,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessMutableChildContentsAre(self, node, name, expected_contents):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         d.addCallback(lambda node: node.download_best_version())
         def _check(contents):
@@ -2641,7 +2649,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessRWChildURIIs(self, node, name, expected_uri):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         def _check(child):
             self.failUnless(child.is_unknown() or not child.is_readonly())
@@ -2654,7 +2662,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessROChildURIIs(self, node, name, expected_uri):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         def _check(child):
             self.failUnless(child.is_unknown() or child.is_readonly())
@@ -2665,7 +2673,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessURIMatchesRWChild(self, got_uri, node, name):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         def _check(child):
             self.failUnless(child.is_unknown() or not child.is_readonly())
@@ -2678,7 +2686,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessURIMatchesROChild(self, got_uri, node, name):
-        assert isinstance(name, unicode)
+        assert isinstance(name, str)
         d = node.get_child_at_path(name)
         def _check(child):
             self.failUnless(child.is_unknown() or child.is_readonly())
@@ -2689,7 +2697,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def failUnlessCHKURIHasContents(self, got_uri, contents):
-        if isinstance(got_uri, unicode):
+        if isinstance(got_uri, str):
             got_uri = got_uri.encode("utf-8")
         self.failUnless(self.get_all_contents()[got_uri] == contents)
 
@@ -2740,7 +2748,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d = self.POST("/uri", t="upload",
                       file=("new.txt", self.NEWFILE_CONTENTS))
         def _check_upload_results(page):
-            page = unicode(page, "utf-8")
+            page = str(page, "utf-8")
             # this should be a page which describes the results of the upload
             # that just finished.
             self.failUnlessIn("Upload Results:", page)
@@ -2828,7 +2836,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                     filecap = mo.group(1)
                 self.failUnless(filecap.startswith(uri_prefix),
                                 (uri_prefix, filecap))
-                return self.GET("/uri/%s?t=json" % unicode(filecap, "utf-8"))
+                return self.GET("/uri/%s?t=json" % str(filecap, "utf-8"))
             d.addCallback(_got_results)
             def _got_json(raw):
                 data = json.loads(raw)
@@ -2860,7 +2868,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                           file=(filename, self.NEWFILE_CONTENTS * 300000))
             def _got_filecap(filecap):
                 if fn is not None:
-                    filenameu = unicode(filename)
+                    filenameu = str(filename)
                     self.failUnlessURIMatchesRWChild(filecap, fn, filenameu)
                 self.failUnless(filecap.startswith(uri_prefix))
                 return self.GET(self.public_url + "/foo/%s?t=json" % filename)
@@ -2959,9 +2967,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check_page_json(res):
             parsed = json.loads(res)
             self.failUnlessEqual(parsed[0], "dirnode")
-            children = dict( [(unicode(name),value)
+            children = dict( [(str(name),value)
                               for (name,value)
-                              in parsed[1]["children"].items()] )
+                              in list(parsed[1]["children"].items())] )
             self.failUnlessIn(u"new.txt", children)
             new_json = children[u"new.txt"]
             self.failUnlessEqual(new_json[0], "filenode")
@@ -3140,7 +3148,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(bar_url, t="check", return_to=redir_url)
-        res = unicode(res, "utf-8")
+        res = str(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file", res)
         self.failUnlessIn(redir_url, res)
@@ -3164,7 +3172,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(bar_url, t="check", return_to=redir_url)
-        res = unicode(res, "utf-8")
+        res = str(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file", res)
         self.failUnlessIn(redir_url, res)
@@ -3182,7 +3190,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     code=http.FOUND)
 
         res = yield self.POST(foo_url, t="check", return_to=redir_url)
-        res = unicode(res, "utf-8")
+        res = str(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file/directory", res)
         self.failUnlessIn(redir_url, res)
@@ -3205,7 +3213,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                     method="post", data=body, headers=headers,
                                     code=http.FOUND)
         res = yield self.POST(foo_url, t="check", return_to=redir_url)
-        res = unicode(res, "utf-8")
+        res = str(res, "utf-8")
         self.failUnlessIn("Healthy :", res)
         self.failUnlessIn("Return to file/directory", res)
         self.failUnlessIn(redir_url, res)
@@ -3216,7 +3224,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check(res):
             self.failUnlessIn(b"Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % unicode(self._quux_txt_uri, "utf-8"))
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % str(self._quux_txt_uri, "utf-8"))
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check"))
         d.addCallback(_check)
@@ -3228,7 +3236,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _check(res):
             self.failUnlessIn(b"Healthy", res)
         d.addCallback(_check)
-        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % unicode(self._quux_txt_uri, "ascii"))
+        quux_extension_url = "/uri/%s" % urlquote("%s:3:131073" % str(self._quux_txt_uri, "ascii"))
         d.addCallback(lambda ignored:
                       self.POST(quux_extension_url, t="check", repair="true"))
         d.addCallback(_check)
@@ -3301,9 +3309,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
 
         foo_si = self._foo_node.get_storage_index()
         foo_si_s = base32.b2a(foo_si)
-        res = yield self.GET("/operations/123/%s?output=JSON" % unicode(foo_si_s, "ascii"))
+        res = yield self.GET("/operations/123/%s?output=JSON" % str(foo_si_s, "ascii"))
         data = json.loads(res)
-        self.failUnlessEqual(data["storage-index"], unicode(foo_si_s, "ascii"))
+        self.failUnlessEqual(data["storage-index"], str(foo_si_s, "ascii"))
         self.failUnless(data["results"]["healthy"])
 
     def test_POST_DIRURL_deepcheck_and_repair(self):
@@ -3329,7 +3337,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check_json)
         d.addCallback(self.get_operation_results, "124", "html")
         def _check_html(res):
-            res = unicode(res, "utf-8")
+            res = str(res, "utf-8")
             self.failUnlessIn("Objects Checked: <span>11</span>", res)
 
             self.failUnlessIn("Objects Healthy (before repair): <span>11</span>", res)
@@ -3391,7 +3399,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
-        d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
+        d.addCallback(self.failUnlessNodeKeysAre, list(newkids.keys()))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         return d
@@ -3443,7 +3451,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"newdir"))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
-        d.addCallback(self.failUnlessNodeKeysAre, newkids.keys())
+        d.addCallback(self.failUnlessNodeKeysAre, list(newkids.keys()))
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
         d.addCallback(self.failUnlessROChildURIIs, u"child-imm", caps['filecap1'])
         d.addCallback(lambda res: self._foo_node.get(u"newdir"))
@@ -3528,7 +3536,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         url = self.webish_url + "/uri?t=mkdir&redirect_to_result=true"
         target = yield self.shouldRedirectTo(url, None, method="post",
                                              code=http.SEE_OTHER)
-        target = urlunquote(unicode(target, "ascii"))
+        target = urlunquote(str(target, "ascii"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     @inlineCallbacks
@@ -3538,7 +3546,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                              method="post",
                                              data=body, headers=headers,
                                              code=http.SEE_OTHER)
-        target = urlunquote(unicode(target, "ascii"))
+        target = urlunquote(str(target, "ascii"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def _make_readonly(self, u):
@@ -3615,7 +3623,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _after_mkdir(res):
             self.failUnless(res.startswith(b"URI:DIR"), res)
             n = self.s.create_node_from_uri(res)
-            d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
+            d2 = self.failUnlessNodeKeysAre(n, list(newkids.keys()))
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                        caps['filecap1']))
@@ -3666,7 +3674,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         def _after_mkdir(res):
             self.failUnless(res.startswith(b"URI:DIR"), res)
             n = self.s.create_node_from_uri(res)
-            d2 = self.failUnlessNodeKeysAre(n, newkids.keys())
+            d2 = self.failUnlessNodeKeysAre(n, list(newkids.keys()))
             d2.addCallback(lambda ign:
                            self.failUnlessROChildURIIs(n, u"child-imm",
                                                           caps['filecap1']))
@@ -3701,7 +3709,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_welcome_page_mkdir_button(self):
         # Fetch the welcome page.
         res = yield self.GET("/")
-        res = unicode(res, "utf-8")
+        res = str(res, "utf-8")
         MKDIR_BUTTON_RE = re.compile(
             '<form(?: action="([^"]*)"| method="post"| enctype="multipart/form-data"){3}>.*'
             '<input (?:type="hidden" |name="t" |value="([^"]*?)" ){3}/>[ ]*'
@@ -3720,7 +3728,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         target = yield self.shouldRedirectTo(url, None,
                                              method="post",
                                              code=http.SEE_OTHER)
-        target = urlunquote(unicode(target, "utf-8"))
+        target = urlunquote(str(target, "utf-8"))
         self.failUnless(target.startswith("uri/URI:DIR2:"), target)
 
     def test_POST_mkdir_replace(self): # return value?
@@ -4041,7 +4049,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                      to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4055,7 +4063,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_new_name(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="wibble.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                      to_name="wibble.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4071,7 +4079,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_replace(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                      to_name="baz.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
@@ -4086,7 +4094,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "There was already a child by that name, and you asked me to not replace it",
                              self.POST, self.public_url + "/foo", t="relink",
                              replace="false", from_name="bar.txt",
-                             to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                             to_name="baz.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
@@ -4098,7 +4106,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_no_replace_explicitly_same_link(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       replace="false", from_name="bar.txt",
-                      to_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
+                      to_name="bar.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
@@ -4109,7 +4117,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_replace_only_files(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       replace="only-files", from_name="bar.txt",
-                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                      to_name="baz.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/baz.txt"))
@@ -4124,7 +4132,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "There was already a child by that name, and you asked me to not replace it.",
                              self.POST, self.public_url + "/foo", t="relink",
                              replace="only-files", from_name="bar.txt",
-                             to_name="sub", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
+                             to_name="sub", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt?t=json"))
@@ -4137,7 +4145,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "to_name= may not contain a slash",
                              self.POST, self.public_url + "/foo", t="relink",
                              from_name="bar.txt",
-                             to_name="slash/fail.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                             to_name="slash/fail.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         d.addCallback(lambda res:
                       self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res:
@@ -4151,13 +4159,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                                        t="relink",
                                        from_name="nope/bar.txt",
                                        to_name="fail.txt",
-                                       to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub"))
+                                       to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub"))
         return d
 
     def test_POST_relink_file_explicitly_same_link(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
+                      to_name="bar.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
         d.addCallback(self.failUnlessIsBarDotTxt)
@@ -4178,7 +4186,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_file_same_dir(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_name="baz.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo")
+                      to_name="baz.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo")
         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.failUnlessNodeHasChild(self._sub_node, u"baz.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
@@ -4193,13 +4201,13 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              self.POST,
                              self.public_url + "/foo", t="relink",
                              replace="boogabooga", from_name="bar.txt",
-                             to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub")
+                             to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub")
         return d
 
     def test_POST_relink_file_multi_level(self):
         d = self.POST2(self.public_url + "/foo/sub/level2?t=mkdir", "")
         d.addCallback(lambda res: self.POST(self.public_url + "/foo", t="relink",
-                      from_name="bar.txt", to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/sub/level2"))
+                      from_name="bar.txt", to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/sub/level2"))
         d.addCallback(lambda res: self.failIfNodeHasChild(self._foo_node, u"bar.txt"))
         d.addCallback(lambda res: self.failIfNodeHasChild(self._sub_node, u"bar.txt"))
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/sub/level2/bar.txt"))
@@ -4224,7 +4232,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                             "404 Not Found", "No such child: nopechucktesta",
                             self.POST, self.public_url + "/foo", t="relink",
                             from_name="bar.txt",
-                            to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/nopechucktesta")
+                            to_dir=str(self.public_root.get_uri(), "utf-8") + "/nopechucktesta")
         return d
 
     def test_POST_relink_file_into_file(self):
@@ -4232,7 +4240,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                              "400 Bad Request", "to_dir is not a directory",
                              self.POST, self.public_url + "/foo", t="relink",
                              from_name="bar.txt",
-                             to_dir=unicode(self.public_root.get_uri(), "utf-8") + "/foo/baz.txt")
+                             to_dir=str(self.public_root.get_uri(), "utf-8") + "/foo/baz.txt")
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/baz.txt"))
         d.addCallback(self.failUnlessIsBazDotTxt)
         d.addCallback(lambda res: self.GET(self.public_url + "/foo/bar.txt"))
@@ -4256,10 +4264,10 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_POST_relink_dir(self):
         d = self.POST(self.public_url + "/foo", t="relink",
                       from_name="bar.txt",
-                      to_dir=unicode(self.public_root.get_uri(), "ascii") + "/foo/empty")
+                      to_dir=str(self.public_root.get_uri(), "ascii") + "/foo/empty")
         d.addCallback(lambda res: self.POST(self.public_url + "/foo",
                       t="relink", from_name="empty",
-                      to_dir=unicode(self.public_root.get_uri(), "ascii") + "/foo/sub"))
+                      to_dir=str(self.public_root.get_uri(), "ascii") + "/foo/sub"))
         d.addCallback(lambda res:
                       self.failIfNodeHasChild(self._foo_node, u"empty"))
         d.addCallback(lambda res:
@@ -4286,12 +4294,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         self.assertIn(response.code, codes)
         location = response.headers.getRawHeaders(b"location")[0]
         if target_location is not None:
-            self.assertEquals(unicode(location, "ascii"), target_location)
+            self.assertEquals(str(location, "ascii"), target_location)
         returnValue(location)
 
     @inlineCallbacks
     def test_GET_URI_form(self):
-        relbase = "/uri?uri=%s" % unicode(self._bar_txt_uri, "ascii")
+        relbase = "/uri?uri=%s" % str(self._bar_txt_uri, "ascii")
         base = self.webish_url + relbase
         # this is supposed to give us a redirect to /uri/$URI, plus arguments
         targetbase = self.webish_url + "/uri/%s" % urlquote(self._bar_txt_uri)
@@ -4305,7 +4313,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         data = yield self.GET(relbase, followRedirect=True)
         self.failUnlessIsBarDotTxt(data)
         self.log(None, "got file by uri, about to get dir by uri")
-        data = yield self.GET("/uri?uri=%s&t=json" % unicode(self._foo_uri, "ascii"),
+        data = yield self.GET("/uri?uri=%s&t=json" % str(self._foo_uri, "ascii"),
                               followRedirect=True)
         self.failUnlessIsFooJSON(data)
         self.log(None, "got dir by uri")
@@ -4339,7 +4347,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return res
 
     def test_GET_URI_URL(self):
-        base = "/uri/%s" % unicode(self._bar_txt_uri, "ascii")
+        base = "/uri/%s" % str(self._bar_txt_uri, "ascii")
         d = self.GET(base)
         d.addCallback(self.failUnlessIsBarDotTxt)
         d.addCallback(lambda res: self.GET(base+"?filename=bar.txt"))
@@ -4349,14 +4357,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         return d
 
     def test_GET_URI_URL_dir(self):
-        base = "/uri/%s?t=json" % unicode(self._foo_uri, "ascii")
+        base = "/uri/%s?t=json" % str(self._foo_uri, "ascii")
         d = self.GET(base)
         d.addCallback(self.failUnlessIsFooJSON)
         return d
 
     @inlineCallbacks
     def test_GET_URI_URL_missing(self):
-        base = "/uri/%s" % unicode(self._bad_file_uri, "ascii")
+        base = "/uri/%s" % str(self._bad_file_uri, "ascii")
         url = self.webish_url + base
         yield self.assertHTTPError(url, http.GONE, "NotEnoughSharesError")
         # TODO: how can we exercise both sides of WebDownloadTarget.fail
@@ -4504,7 +4512,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessIn(uri, self.get_all_contents())
             self.failUnlessReallyEqual(self.get_all_contents()[uri],
                                        file_contents)
-            return self.GET("/uri/%s" % unicode(uri, "utf-8"))
+            return self.GET("/uri/%s" % str(uri, "utf-8"))
         d.addCallback(_check)
         def _check2(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4519,7 +4527,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessIn(uri, self.get_all_contents())
             self.failUnlessReallyEqual(self.get_all_contents()[uri],
                                        file_contents)
-            return self.GET("/uri/%s" % unicode(uri, "utf-8"))
+            return self.GET("/uri/%s" % str(uri, "utf-8"))
         d.addCallback(_check)
         def _check2(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4548,7 +4556,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         d.addCallback(_check1)
         def _check2(data):
             self.failUnlessReallyEqual(data, file_contents)
-            return self.GET("/uri/%s" % urlquote(unicode(self.filecap, "utf-8")))
+            return self.GET("/uri/%s" % urlquote(str(self.filecap, "utf-8")))
         d.addCallback(_check2)
         def _check3(res):
             self.failUnlessReallyEqual(res, file_contents)
@@ -4561,7 +4569,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             n = self.s.create_node_from_uri(uri.strip())
             d2 = self.failUnlessNodeKeysAre(n, [])
             d2.addCallback(lambda res:
-                           self.GET("/uri/%s?t=json" % unicode(uri, "utf-8")))
+                           self.GET("/uri/%s?t=json" % str(uri, "utf-8")))
             return d2
         d.addCallback(_check)
         d.addCallback(self.failUnlessIsEmptyJSON)
@@ -4616,7 +4624,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.new_data = new_data
         d.addCallback(_then)
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?replace=True&offset=100" % unicode(self.filecap, "utf-8"),
+            self.PUT("/uri/%s?replace=True&offset=100" % str(self.filecap, "utf-8"),
                      b"replaced and so on"))
         def _get_data(filecap):
             n = self.s.create_node_from_uri(filecap)
@@ -4626,14 +4634,14 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             self.failUnlessEqual(results, self.new_data))
         # Now try appending things to the file
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?offset=%d" % (unicode(self.filecap, "utf-8"), len(self.new_data)),
+            self.PUT("/uri/%s?offset=%d" % (str(self.filecap, "utf-8"), len(self.new_data)),
                      b"puppies" * 100))
         d.addCallback(_get_data)
         d.addCallback(lambda results:
             self.failUnlessEqual(results, self.new_data + (b"puppies" * 100)))
         # and try replacing the beginning of the file
         d.addCallback(lambda ignored:
-            self.PUT("/uri/%s?offset=0" % unicode(self.filecap, "utf-8"), b"begin"))
+            self.PUT("/uri/%s?offset=0" % str(self.filecap, "utf-8"), b"begin"))
         d.addCallback(_get_data)
         d.addCallback(lambda results:
             self.failUnlessEqual(results, b"begin"+self.new_data[len(b"begin"):]+(b"puppies"*100)))
@@ -4644,7 +4652,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
         file_contents = b"test file" * 100000 # about 900 KiB
         filecap = yield self.PUT("/uri?mutable=true", file_contents)
         # Negative offsets should cause an error.
-        url = self.webish_url + "/uri/%s?offset=-1" % unicode(filecap, "utf-8")
+        url = self.webish_url + "/uri/%s?offset=-1" % str(filecap, "utf-8")
         yield self.assertHTTPError(url, 400, "Invalid offset",
                                    method="put", data=b"foo")
 
@@ -4652,7 +4660,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
     def test_PUT_update_at_offset_immutable(self):
         file_contents = b"Test file" * 100000
         filecap = yield self.PUT("/uri", file_contents)
-        url = self.webish_url + "/uri/%s?offset=50" % unicode(filecap, "utf-8")
+        url = self.webish_url + "/uri/%s?offset=50" % str(filecap, "utf-8")
         yield self.assertHTTPError(url, 400, "immutable",
                                    method="put", data=b"foo")
 
@@ -4803,7 +4811,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             dircap_uri,
         )
         self.assertEqual(
-            unicode(response.request.absoluteURI, "utf-8"),
+            str(response.request.absoluteURI, "utf-8"),
             self.webish_url + "/uri/{}?t=json".format(urlquote(dircap)))
         if response.code >= 400:
             raise Error(response.code, response=response.content())
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 42e2ec5fb..0c41bb742 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -194,5 +194,6 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_root",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",
+    "allmydata.test.web.test_web",
     "allmydata.test.web.test_webish",
 ]

From 51bd4b719c0c78aaaa740605adbdf410844a55e5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 2 Feb 2021 15:28:12 -0500
Subject: [PATCH 36/50] Fix lint.

---
 src/allmydata/test/web/test_web.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 4dbdf685b..cebe709c1 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -4290,7 +4290,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
                  http.FOUND,
                  http.TEMPORARY_REDIRECT,
                  ] if code is None else [code]
-        body = yield response.content()
+        yield response.content()
         self.assertIn(response.code, codes)
         location = response.headers.getRawHeaders(b"location")[0]
         if target_location is not None:

From 965a3e3f051e0f451d62b575819f1f7d4497baa4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 2 Feb 2021 15:28:25 -0500
Subject: [PATCH 37/50] News file.

---
 newsfragments/3596.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3596.minor

diff --git a/newsfragments/3596.minor b/newsfragments/3596.minor
new file mode 100644
index 000000000..e69de29bb

From c26f28584e802104fea888ea5f49659280e5d437 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Feb 2021 10:11:30 -0500
Subject: [PATCH 38/50] Clarify what it does.

---
 src/allmydata/util/jsonbytes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/util/jsonbytes.py b/src/allmydata/util/jsonbytes.py
index ab9d5fac0..935187d29 100644
--- a/src/allmydata/util/jsonbytes.py
+++ b/src/allmydata/util/jsonbytes.py
@@ -17,7 +17,7 @@ import json
 
 
 def _bytes_to_unicode(obj):
-    """Convert keys of dicts from bytes to unicode, recursively."""
+    """Convert any bytes objects to unicode, recursively."""
     if isinstance(obj, bytes):
         return obj.decode("utf-8")
     if isinstance(obj, dict):

From a1add9a5121121eb1896d2ec25e9a21908b7b9f6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Feb 2021 10:13:19 -0500
Subject: [PATCH 39/50] Clarify.

---
 src/allmydata/test/common_util.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 0bff5e171..f898e75b2 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -203,7 +203,10 @@ def flip_one_bit(s, offset=0, size=None):
 class ReallyEqualMixin(object):
     def failUnlessReallyEqual(self, a, b, msg=None):
         self.assertEqual(a, b, msg)
-        # Make sure unicode strings are a consistent type:
+        # Make sure unicode strings are a consistent type. Specifically there's
+        # Future newstr (backported Unicode type) vs. Python 2 native unicode
+        # type. They're equal, and _logically_ the same type, but have
+        # different types in practice.
         if a.__class__ == future_str:
             a = unicode(a)
         if b.__class__ == future_str:

From 2595e25258315f79c4eb82785770767716f247db Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Feb 2021 10:16:34 -0500
Subject: [PATCH 40/50] Improve coverage, specifically of values that are
 bytes.

---
 src/allmydata/test/test_util.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py
index 58de96d1c..5f5db82bd 100644
--- a/src/allmydata/test/test_util.py
+++ b/src/allmydata/test/test_util.py
@@ -491,12 +491,16 @@ class JSONBytes(unittest.TestCase):
     """Tests for BytesJSONEncoder."""
 
     def test_encode_bytes(self):
-        """BytesJSONEncoder can encode bytes."""
+        """BytesJSONEncoder can encode bytes.
+
+        Bytes are presumed to be UTF-8 encoded.
+        """
+        snowman = u"def\N{SNOWMAN}\uFF00"
         data = {
-            b"hello": [1, b"cd", {b"abc": 123}],
+            b"hello": [1, b"cd", {b"abc": [123, snowman.encode("utf-8")]}],
         }
         expected = {
-            u"hello": [1, u"cd", {u"abc": 123}],
+            u"hello": [1, u"cd", {u"abc": [123, snowman]}],
         }
         # Bytes get passed through as if they were UTF-8 Unicode:
         encoded = jsonbytes.dumps(data)

From 92058e04b98055a763ccedba36b50071cbf8cc81 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Feb 2021 10:19:41 -0500
Subject: [PATCH 41/50] Make sure typechecks run under Python 3.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 915981e0c..2ff83cbd4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -114,6 +114,7 @@ commands =
 
 
 [testenv:typechecks]
+basepython = python3
 skip_install = True
 deps =
     mypy

From cb99b27b78c3476e9d40fb739b38f2f48a68d7b5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Feb 2021 10:21:41 -0500
Subject: [PATCH 42/50] Nothing to see here, move along.

---
 src/allmydata/webish.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index 2d9ef25ff..0dc51a635 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -60,12 +60,13 @@ else:
         enables setting the filename not via the MIME filename, but via a
         separate field called "name".
 
-        Thus we need to do this ridiculous workaround.
+        Thus we need to do this ridiculous workaround.  Mypy doesn't like it
+        either, thus the ``# type: ignore`` below.
 
         Source for idea:
         https://mail.python.org/pipermail/python-dev/2017-February/147402.html
         """
-        @property
+        @property  # type: ignore
         def filename(self):
             if self.name == "file" and not self._mime_filename:
                 # We use the file field to upload files, see directory.py's

From 5568170c243ab717221571ca61a48c9a717dadbf Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 11 Feb 2021 15:46:04 -0500
Subject: [PATCH 43/50] Slightly better docs for the share count limits on
 convergence hash tag

---
 src/allmydata/util/hashutil.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py
index f82c04efd..8525dd95e 100644
--- a/src/allmydata/util/hashutil.py
+++ b/src/allmydata/util/hashutil.py
@@ -180,8 +180,8 @@ def _convergence_hasher_tag(k, n, segsize, convergence):
     """
     Create the convergence hashing tag.
 
-    :param int k: Required shares.
-    :param int n: Total shares.
+    :param int k: Required shares (in [1..256]).
+    :param int n: Total shares (in [1..256]).
     :param int segsize: Maximum segment size.
     :param bytes convergence: The convergence secret.
 
@@ -193,10 +193,17 @@ def _convergence_hasher_tag(k, n, segsize, convergence):
             "k > n not allowed; k = {}, n = {}".format(k, n),
         )
     if k < 1 or n < 1:
+        # It doesn't make sense to have zero shares.  Zero shares carry no
+        # information, cannot encode any part of the application data.
         raise ValueError(
             "k, n < 1 not allowed; k = {}, n = {}".format(k, n),
         )
     if k > 256 or n > 256:
+        # ZFEC supports encoding application data into a maximum of 256
+        # shares.  If we ignore the limitations of ZFEC, it may be fine to use
+        # a configuration with more shares than that and it may be fine to
+        # construct a convergence tag from such a configuration.  Since ZFEC
+        # is the only supported encoder, though, this is moot for now.
         raise ValueError(
             "k, n > 256 not allowed; k = {}, n = {}".format(k, n),
         )

From 451ede2666318cec793c70fd7ce9d279a20d6556 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 11 Feb 2021 15:58:28 -0500
Subject: [PATCH 44/50] news fragment

---
 newsfragments/3326.installation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3326.installation

diff --git a/newsfragments/3326.installation b/newsfragments/3326.installation
new file mode 100644
index 000000000..2a3a64e32
--- /dev/null
+++ b/newsfragments/3326.installation
@@ -0,0 +1 @@
+Debian 8 support has been replaced with Debian 10 support.

From 7f224414233c2c82317c1bf2bda005602787d327 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 11 Feb 2021 15:59:51 -0500
Subject: [PATCH 45/50] Change Debian 8 to Debian 10 for CI

---
 .circleci/config.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 29b55ad5f..42e47957c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,7 +29,7 @@ workflows:
       - "debian-9": &DOCKERHUB_CONTEXT
           context: "dockerhub-auth"
 
-      - "debian-8":
+      - "debian-10":
           <<: *DOCKERHUB_CONTEXT
           requires:
             - "debian-9"
@@ -107,7 +107,7 @@ workflows:
                 - "master"
 
     jobs:
-      - "build-image-debian-8":
+      - "build-image-debian-10":
           <<: *DOCKERHUB_CONTEXT
       - "build-image-debian-9":
           <<: *DOCKERHUB_CONTEXT
@@ -277,11 +277,11 @@ jobs:
             fi
 
 
-  debian-8:
+  debian-10:
     <<: *DEBIAN
     docker:
       - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/debian:8-py2.7"
+        image: "tahoelafsci/debian:10-py2.7"
         user: "nobody"
 
 
@@ -529,12 +529,12 @@ jobs:
             docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
 
 
-  build-image-debian-8:
+  build-image-debian-10:
     <<: *BUILD_IMAGE
 
     environment:
       DISTRO: "debian"
-      TAG: "8"
+      TAG: "10"
       PYTHON_VERSION: "2.7"
 
 

From a8b1c204d2b836e70d84f8ae9179fba523648059 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 11 Feb 2021 16:06:18 -0500
Subject: [PATCH 46/50] Mark the expected result literal as the correct type,
 bytes

---
 src/allmydata/test/test_hashutil.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py
index 6ac73ae60..482e79c0b 100644
--- a/src/allmydata/test/test_hashutil.py
+++ b/src/allmydata/test/test_hashutil.py
@@ -133,9 +133,9 @@ class HashUtilTests(unittest.TestCase):
         convergence secret.
         """
         self.assertEqual(
-            "allmydata_immutable_content_to_key_with_added_secret_v1+"
-            "16:\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42,"
-            "9:3,10,1024,",
+            b"allmydata_immutable_content_to_key_with_added_secret_v1+"
+            b"16:\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42\x42,"
+            b"9:3,10,1024,",
             hashutil._convergence_hasher_tag(
                 k=3,
                 n=10,

From 9a9b4bb232549c0802c74593454e2451a62d9e3d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 12 Feb 2021 10:14:23 -0500
Subject: [PATCH 47/50] Use a monkey patching library for monkey patching

---
 src/allmydata/test/eliotutil.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index f91d76e03..2a5c85ebf 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -36,6 +36,10 @@ from eliot.testing import (
     check_for_errors,
 )
 
+from twisted.python.monkey import (
+    MonkeyPatcher,
+)
+
 from ..util.jsonbytes import BytesJSONEncoder
 
 
@@ -115,13 +119,16 @@ class EliotLoggedRunTest(object):
         # and things do not go well when we add too much extra behavior here.
         # For example, exceptions raised here often just kill the whole
         # runner.
+        patcher = MonkeyPatcher()
 
         # So, grab the test method.
         name = self.case._testMethodName
         original = getattr(self.case, name)
+        decorated = with_logging(ensure_text(self.case.id()), original)
+        patcher.addPatch(self.case, name, decorated)
         try:
-            # Patch in a decorated version of it.
-            setattr(self.case, name, with_logging(ensure_text(self.case.id()), original))
+            # Patch it in
+            patcher.patch()
             # Then use the rest of the machinery to run it.
             return self._run_tests_with_factory(
                 self.case,
@@ -130,7 +137,7 @@ class EliotLoggedRunTest(object):
             ).run(result)
         finally:
             # Clean up the patching for idempotency or something.
-            delattr(self.case, name)
+            patcher.restore()
 
 
 def with_logging(

From 585c554081be261daa43b8750819b74029c65252 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 12 Feb 2021 10:22:17 -0500
Subject: [PATCH 48/50] Clarify _TwoLoggers

---
 src/allmydata/test/eliotutil.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index 2a5c85ebf..63c24f08a 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -26,10 +26,15 @@ from functools import (
 
 import attr
 
+from zope.interface import (
+    implementer,
+)
+
 from eliot import (
     ActionType,
     Field,
     MemoryLogger,
+    ILogger,
 )
 from eliot.testing import (
     swap_logger,
@@ -170,6 +175,7 @@ def with_logging(
     return run_with_logging
 
 
+@implementer(ILogger)
 class _TwoLoggers(object):
     """
     Log to two loggers.
@@ -180,9 +186,13 @@ class _TwoLoggers(object):
     the normal places and validate all written log messages.
     """
     def __init__(self, a, b):
-        self._a = a
-        self._b = b
+        """
+        :param ILogger a: One logger
+        :param ILogger b: Another logger
+        """
+        self._a = a # type: ILogger
+        self._b = b # type: ILogger
 
-    def write(self, *args, **kwargs):
-        self._a.write(*args, **kwargs)
-        self._b.write(*args, **kwargs)
+    def write(self, dictionary, serializer=None):
+        self._a.write(dictionary, serializer)
+        self._b.write(dictionary, serializer)

From 5aa452c8bbbb0e6a9a7e03e77859830d1b2adc4a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 12 Feb 2021 10:23:07 -0500
Subject: [PATCH 49/50] Drop build-porting-depgraph step.

---
 .circleci/config.yml | 32 --------------------------------
 1 file changed, 32 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 42e47957c..ad5360082 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -86,11 +86,6 @@ workflows:
             # integration tests.
             - "debian-9"
 
-      # Generate the underlying data for a visualization to aid with Python 3
-      # porting.
-      - "build-porting-depgraph":
-          <<: *DOCKERHUB_CONTEXT
-
       - "typechecks":
           <<: *DOCKERHUB_CONTEXT
 
@@ -451,33 +446,6 @@ jobs:
             # them in parallel.
             nix-build --cores 3 --max-jobs 2 nix/
 
-  # Generate up-to-date data for the dependency graph visualizer.
-  build-porting-depgraph:
-    # Get a system in which we can easily install Tahoe-LAFS and all its
-    # dependencies.  The dependency graph analyzer works by executing the code.
-    # It's Python, what do you expect?
-    <<: *DEBIAN
-
-    steps:
-      - "checkout"
-
-      - add_ssh_keys:
-          fingerprints:
-            # Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
-            # This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
-            - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
-
-      - run:
-          name: "Setup Python Environment"
-          command: |
-            /tmp/venv/bin/pip install -e /tmp/project
-
-      - run:
-          name: "Generate dependency graph data"
-          command: |
-            . /tmp/venv/bin/activate
-            ./misc/python3/depgraph.sh
-
   typechecks:
     docker:
       - <<: *DOCKERHUB_AUTH

From 66b5a1577acd7d3757e4a8ceb638240c4ec45b87 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 12 Feb 2021 10:23:20 -0500
Subject: [PATCH 50/50] News file.

---
 newsfragments/3612.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3612.minor

diff --git a/newsfragments/3612.minor b/newsfragments/3612.minor
new file mode 100644
index 000000000..e69de29bb