From 1c9e4ec842ccb42f670733071e2ac9ed869917fc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:04:52 -0400
Subject: [PATCH 01/39] Move connection_status tests into their own module.

---
 src/allmydata/test/test_connection_status.py | 112 +++++++++++++++++++
 src/allmydata/test/test_connections.py       | 104 -----------------
 2 files changed, 112 insertions(+), 104 deletions(-)
 create mode 100644 src/allmydata/test/test_connection_status.py

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
new file mode 100644
index 000000000..763c14c57
--- /dev/null
+++ b/src/allmydata/test/test_connection_status.py
@@ -0,0 +1,112 @@
+"""
+Tests for allmydata.util.connection_status.
+"""
+
+import mock
+
+from twisted.trial import unittest
+
+from ..util import connection_status
+
+class Status(unittest.TestCase):
+    def test_hint_statuses(self):
+        ncs = connection_status._hint_statuses(["h2","h1"],
+                                               {"h1": "hand1", "h4": "hand4"},
+                                               {"h1": "st1", "h2": "st2",
+                                                "h3": "st3"})
+        self.assertEqual(ncs, {"h1 via hand1": "st1",
+                               "h2": "st2"})
+
+    def test_reconnector_connected(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_others(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_listener(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.listenerStatus = ("listener1", "successful")
+        ci.winningHint = None
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected via listener (listener1)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connecting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "connecting"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary, "Trying to connect")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_waiting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "waiting"
+        ri.lastAttempt = 10
+        ri.nextAttempt = 20
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        with mock.patch("time.time", return_value=12):
+            cs = connection_status.from_foolscap_reconnector(rc, 5)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary,
+                         "Reconnecting in 8 seconds (last attempt 2s ago)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 5)
diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py
index 3e2806dd0..9b5bd7f30 100644
--- a/src/allmydata/test/test_connections.py
+++ b/src/allmydata/test/test_connections.py
@@ -7,7 +7,6 @@ from foolscap.connections import tcp
 from ..node import PrivacyError, config_from_string
 from ..node import create_connection_handlers
 from ..node import create_main_tub, _tub_portlocation
-from ..util import connection_status
 from ..util.i2p_provider import create as create_i2p_provider
 from ..util.tor_provider import create as create_tor_provider
 
@@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
             str(ctx.exception),
             "tub.location includes tcp: hint",
         )
-
-class Status(unittest.TestCase):
-    def test_hint_statuses(self):
-        ncs = connection_status._hint_statuses(["h2","h1"],
-                                               {"h1": "hand1", "h4": "hand4"},
-                                               {"h1": "st1", "h2": "st2",
-                                                "h3": "st3"})
-        self.assertEqual(ncs, {"h1 via hand1": "st1",
-                               "h2": "st2"})
-
-    def test_reconnector_connected(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_others(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_listener(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.listenerStatus = ("listener1", "successful")
-        ci.winningHint = None
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected via listener (listener1)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connecting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "connecting"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary, "Trying to connect")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_waiting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "waiting"
-        ri.lastAttempt = 10
-        ri.nextAttempt = 20
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        with mock.patch("time.time", return_value=12):
-            cs = connection_status.from_foolscap_reconnector(rc, 5)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary,
-                         "Reconnecting in 8 seconds (last attempt 2s ago)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 5)

From 243d02ecb1a01216fc16331ec25a04186dfce063 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:08:51 -0400
Subject: [PATCH 02/39] Port to Python 3.

---
 src/allmydata/test/test_connection_status.py | 10 ++++++++++
 src/allmydata/util/_python3.py               |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 763c14c57..2bd8bf6ab 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -1,6 +1,16 @@
 """
 Tests for allmydata.util.connection_status.
+
+Port to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import mock
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 091c248af..f074927b6 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -50,6 +50,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",

From a08cde9a4da3bf636560fc72e2297a371953b38e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:30:16 -0400
Subject: [PATCH 03/39] Port to Python 3.

---
 src/allmydata/interfaces.py             |  1 +
 src/allmydata/util/_python3.py          |  1 +
 src/allmydata/util/connection_status.py | 22 ++++++++++++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 36eb55bfb..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,3 +1,4 @@
+from past.builtins import long
 
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f074927b6..494d5e161 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
     "allmydata.util.gcutil",
diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index 44c12f220..0e8595e81 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -1,3 +1,18 @@
+"""
+Parse connection status from Foolscap.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus
@@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
 
 def from_foolscap_reconnector(rc, last_received):
     ri = rc.getReconnectionInfo()
-    # See foolscap/reconnector.py, ReconnectionInfo, for details about
-    # possible states.
+    # See foolscap/reconnector.py, ReconnectionInfo, for details about possible
+    # states. The returned result is a native string, it seems, so convert to
+    # unicode.
     state = ri.state
+    if isinstance(state, bytes):  # Python 2
+        state = str(state, "ascii")
     if state == "unstarted":
         return ConnectionStatus.unstarted()
 

From e24c21bef779de6d1d419b2135231f628a0bcc20 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:38:58 -0400
Subject: [PATCH 04/39] Make configutil tests more standalone, and less
 repetitive.

---
 src/allmydata/test/cli/test_create.py | 27 ++++++++
 src/allmydata/test/test_configutil.py | 90 +++++++--------------------
 2 files changed, 49 insertions(+), 68 deletions(-)

diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py
index 75162c39e..f013c0205 100644
--- a/src/allmydata/test/cli/test_create.py
+++ b/src/allmydata/test/cli/test_create.py
@@ -6,6 +6,8 @@ from twisted.python import usage
 from allmydata.util import configutil
 from ..common_util import run_cli, parse_cli
 from ...scripts import create_node
+from ... import client
+
 
 def read_config(basedir):
     tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@@ -33,6 +35,31 @@ class Config(unittest.TestCase):
             e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
             self.assertIn("option %s not recognized" % (option,), str(e))
 
+    def test_create_client_config(self):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+
+        with open(fname, 'w') as f:
+            opts = {"nickname": "nick",
+                    "webport": "tcp:3456",
+                    "hide-ip": False,
+                    "listen": "none",
+                    "shares-needed": "1",
+                    "shares-happy": "1",
+                    "shares-total": "1",
+                    }
+            create_node.write_node_config(f, opts)
+            create_node.write_client_config(f, opts)
+
+        config = configutil.get_config(fname)
+        # should succeed, no exceptions
+        configutil.validate_config(
+            fname,
+            config,
+            client._valid_config(),
+        )
+
     @defer.inlineCallbacks
     def test_client(self):
         basedir = self.mktemp()
diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index 45eb6ac25..ffb5f5320 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -3,12 +3,9 @@ import os.path
 from twisted.trial import unittest
 
 from allmydata.util import configutil
-from allmydata.test.no_network import GridTestMixin
-from ..scripts import create_node
-from .. import client
 
 
-class ConfigUtilTests(GridTestMixin, unittest.TestCase):
+class ConfigUtilTests(unittest.TestCase):
     def setUp(self):
         super(ConfigUtilTests, self).setUp()
         self.static_valid_config = configutil.ValidConfiguration(
@@ -20,10 +17,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
         )
 
+    def create_tahoe_cfg(self, cfg):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+        with open(fname, "w") as f:
+            f.write(cfg)
+        return fname
+
     def test_config_utils(self):
-        self.basedir = "cli/ConfigUtilTests/test-config-utils"
-        self.set_up_grid(oneshare=True)
-        tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
+        tahoe_cfg = self.create_tahoe_cfg("""\
+[node]
+nickname = client-0
+web.port = adopt-socket:fd=5
+[storage]
+enabled = false
+""")
 
         # test that at least one option was read correctly
         config = configutil.get_config(tahoe_cfg)
@@ -45,12 +54,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
 
     def test_config_validation_success(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -66,12 +70,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         validation but are matched by the dynamic validation is considered
         valid.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -82,12 +81,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         )
 
     def test_config_validation_invalid_item(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -103,12 +97,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -124,12 +113,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -145,12 +129,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section, item pair that is matched by neither the
         static nor dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -160,28 +139,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             self.dynamic_valid_config,
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
-
-    def test_create_client_config(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            opts = {"nickname": "nick",
-                    "webport": "tcp:3456",
-                    "hide-ip": False,
-                    "listen": "none",
-                    "shares-needed": "1",
-                    "shares-happy": "1",
-                    "shares-total": "1",
-                    }
-            create_node.write_node_config(f, opts)
-            create_node.write_client_config(f, opts)
-
-        config = configutil.get_config(fname)
-        # should succeed, no exceptions
-        configutil.validate_config(
-            fname,
-            config,
-            client._valid_config(),
-        )

From babe2dbc85d8c6d87bec5e1a919faf71f854b596 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:45:52 -0400
Subject: [PATCH 05/39] Port to Python 3.

---
 src/allmydata/test/test_configutil.py | 15 +++++++++++++++
 src/allmydata/util/_python3.py        |  1 +
 2 files changed, 16 insertions(+)

diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index ffb5f5320..c57381289 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -1,3 +1,18 @@
+"""
+Tests for allmydata.util.configutil.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Omitted dict, cause worried about interactions.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 
 from twisted.trial import unittest
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 494d5e161..b6c4b6c87 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -51,6 +51,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_configutil",
     "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",

From 11b934120c1b5f279e4c84c1d94de8bec9ea2bc8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:49:59 -0400
Subject: [PATCH 06/39] Port to Python 3.

---
 src/allmydata/util/_python3.py   |  1 +
 src/allmydata/util/configutil.py | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index b6c4b6c87..947fcf00b 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.configutil",
     "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 3699db35d..295605b65 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,5 +1,13 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
-from ConfigParser import SafeConfigParser
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from configparser import SafeConfigParser
 
 import attr
 
@@ -13,11 +21,7 @@ class UnknownConfigError(Exception):
 
 def get_config(tahoe_cfg):
     config = SafeConfigParser()
-    with open(tahoe_cfg, "rb") as f:
-        # Skip any initial Byte Order Mark. Since this is an ordinary file, we
-        # don't need to handle incomplete reads, and can assume seekability.
-        if f.read(3) != '\xEF\xBB\xBF':
-            f.seek(0)
+    with open(tahoe_cfg, "r") as f:
         config.readfp(f)
     return config
 
@@ -28,7 +32,7 @@ def set_config(config, section, option, value):
     assert config.get(section, option) == value
 
 def write_config(tahoe_cfg, config):
-    with open(tahoe_cfg, "wb") as f:
+    with open(tahoe_cfg, "w") as f:
         config.write(f)
 
 def validate_config(fname, cfg, valid_config):

From 277298050f16ab3794868f377b6a617820baefe7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:51:27 -0400
Subject: [PATCH 07/39] News file and ratchet.

---
 misc/python3/ratchet-passing | 13 +++++++++++++
 newsfragments/3377.minor     |  0
 2 files changed, 13 insertions(+)
 create mode 100644 newsfragments/3377.minor

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bbc6e6c7e..e6b0d60c2 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -24,6 +24,19 @@ allmydata.test.test_base62.Base62.test_known_values
 allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
 allmydata.test.test_base62.Base62.test_odd_sizes
 allmydata.test.test_base62.Base62.test_roundtrip
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success
+allmydata.test.test_configutil.ConfigUtilTests.test_config_utils
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success
+allmydata.test.test_connection_status.Status.test_hint_statuses
+allmydata.test.test_connection_status.Status.test_reconnector_connected
+allmydata.test.test_connection_status.Status.test_reconnector_connected_listener
+allmydata.test.test_connection_status.Status.test_reconnector_connected_others
+allmydata.test.test_connection_status.Status.test_reconnector_connecting
+allmydata.test.test_connection_status.Status.test_reconnector_waiting
 allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_key_serialization
diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor
new file mode 100644
index 000000000..e69de29bb

From 10378541d72a6853b4cc1f3c86482979f14b3827 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 16:40:00 -0400
Subject: [PATCH 08/39] Use Python 2 ConfigParser, so correct exceptions get
 raised.

---
 src/allmydata/util/configutil.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 295605b65..79980b006 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -7,10 +7,17 @@ from future.utils import PY2
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from configparser import SafeConfigParser
+if PY2:
+    # In theory on Python 2 configparser also works, but then code gets the
+    # wrong exceptions and they don't get handled. So just use native parser
+    # for now.
+    from ConfigParser import SafeConfigParser
+else:
+    from configparser import SafeConfigParser
 
 import attr
 
+
 class UnknownConfigError(Exception):
     """
     An unknown config item was found.

From af61571fa61616e4bdd43326919075d46eeaf6cc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 15:56:08 -0400
Subject: [PATCH 09/39] Accidentally passing test :(

---
 misc/python3/ratchet-passing | 1 +
 1 file changed, 1 insertion(+)

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bf56e0de2..01e58a2fa 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -206,6 +206,7 @@ allmydata.test.test_statistics.Statistics.test_pr_file_loss
 allmydata.test.test_statistics.Statistics.test_repair_cost
 allmydata.test.test_statistics.Statistics.test_repair_count_pmf
 allmydata.test.test_statistics.Statistics.test_survival_pmf
+allmydata.test.test_stats.CPUUsage.test_monitor
 allmydata.test.test_time_format.TimeFormat.test_epoch
 allmydata.test.test_time_format.TimeFormat.test_epoch_in_London
 allmydata.test.test_time_format.TimeFormat.test_format_delta

From b0c4f6d2abaea2d2a2ca5408072cb8ba68fb827a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 16:30:27 -0400
Subject: [PATCH 10/39] Fix Python 2 tests.

---
 src/allmydata/util/configutil.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 79980b006..1a1a93f18 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,3 +1,10 @@
+"""
+Read/write config files.
+
+Configuration is returned as native strings.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -5,7 +12,9 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # We don't do open(), because we want files to read/write native strs when
+    # we do "r" or "w".
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 if PY2:
     # In theory on Python 2 configparser also works, but then code gets the
@@ -27,8 +36,17 @@ class UnknownConfigError(Exception):
 
 
 def get_config(tahoe_cfg):
+    """Load the config, returning a SafeConfigParser.
+
+    Configuration is returned as native strings.
+    """
     config = SafeConfigParser()
     with open(tahoe_cfg, "r") as f:
+        # On Python 2, where we read in bytes, skip any initial Byte Order
+        # Mark. Since this is an ordinary file, we don't need to handle
+        # incomplete reads, and can assume seekability.
+        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
+            f.seek(0)
         config.readfp(f)
     return config
 

From c66d38e1912063a437191a39c0e0b63188e22275 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 18 Aug 2020 08:37:59 -0400
Subject: [PATCH 11/39] Make web.directory.DeepSizeResults a
 MultiFormatResource

---
 src/allmydata/web/directory.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py
index a5ba3bed5..949dbae46 100644
--- a/src/allmydata/web/directory.py
+++ b/src/allmydata/web/directory.py
@@ -53,7 +53,6 @@ from allmydata.web.common import (
     get_mutable_type,
     get_filenode_metadata,
     render_time,
-    MultiFormatPage,
     MultiFormatResource,
     SlotsSequenceElement,
 )
@@ -1213,7 +1212,7 @@ class ManifestElement(ReloadableMonitorElement):
 
 class ManifestResults(MultiFormatResource, ReloadMixin):
 
-    # Control MultiFormatPage
+    # Control MultiFormatResource
     formatArgument = "output"
     formatDefault = "html"
 
@@ -1268,8 +1267,9 @@ class ManifestResults(MultiFormatResource, ReloadMixin):
         return json.dumps(status, indent=1)
 
 
-class DeepSizeResults(MultiFormatPage):
-    # Control MultiFormatPage
+class DeepSizeResults(MultiFormatResource):
+
+    # Control MultiFormatResource
     formatArgument = "output"
     formatDefault = "html"
 

From 0de92a22656a62355180225f8a5fb8661f845545 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 18 Aug 2020 08:38:55 -0400
Subject: [PATCH 12/39] Add newsfragment

---
 newsfragments/3381.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3381.minor

diff --git a/newsfragments/3381.minor b/newsfragments/3381.minor
new file mode 100644
index 000000000..e69de29bb

From e5b98520817a79ddc462d862052fc7c4a20add8c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 13:05:25 -0400
Subject: [PATCH 13/39] News file.

---
 newsfragments/3387.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3387.minor

diff --git a/newsfragments/3387.minor b/newsfragments/3387.minor
new file mode 100644
index 000000000..e69de29bb

From 3d05f6cfafccfd6123744bfc5728c9dab839b362 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 13:06:00 -0400
Subject: [PATCH 14/39] Support multiple venvs.

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 8191c173b..99f905526 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-venv
+venv*
 
 # vim swap files
 *.swp

From be9f02cb13b2bab91058f7f71d49f2b0f40bd056 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 13:15:24 -0400
Subject: [PATCH 15/39] Should be explicitly bytes.

---
 src/allmydata/storage/mutable.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py
index 287ed8fb9..bba501693 100644
--- a/src/allmydata/storage/mutable.py
+++ b/src/allmydata/storage/mutable.py
@@ -48,8 +48,9 @@ class MutableShareFile(object):
     # our sharefiles share with a recognizable string, plus some random
     # binary data to reduce the chance that a regular text file will look
     # like a sharefile.
-    MAGIC = "Tahoe mutable container v1\n" + "\x75\x09\x44\x03\x8e"
+    MAGIC = b"Tahoe mutable container v1\n" + b"\x75\x09\x44\x03\x8e"
     assert len(MAGIC) == 32
+    assert isinstance(MAGIC, bytes)
     MAX_SIZE = MAX_MUTABLE_SHARE_SIZE
     # TODO: decide upon a policy for max share size
 

From 9bdc085c26a21f25810bd8bca0be2d3b55bf230f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 20 Aug 2020 13:20:56 -0400
Subject: [PATCH 16/39] news fragment

---
 newsfragments/3388.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3388.minor

diff --git a/newsfragments/3388.minor b/newsfragments/3388.minor
new file mode 100644
index 000000000..e69de29bb

From 60759597f37e9748eb3ccf3ca753f96034086afd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 20 Aug 2020 13:21:48 -0400
Subject: [PATCH 17/39] Ask for a particular version of Python

In particular, a version Tahoe-LAFS supports.
---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9f7381f33..c15eb1746 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -508,6 +508,7 @@ jobs:
     environment:
       DISTRO: "ubuntu"
       TAG: "20.04"
+      PYTHON_VERSION: "2.7"
 
 
   build-image-centos-8:

From 9d34ab587a009e96b5386b32be72be4a02a9da95 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:17:06 -0400
Subject: [PATCH 18/39] test_storage_web now runnable on Python 3 (even if it
 doesn't pass).

---
 src/allmydata/test/test_storage_web.py |   4 +-
 src/allmydata/web/common.py            |  98 ++------------------
 src/allmydata/web/common_py3.py        | 120 +++++++++++++++++++++++++
 src/allmydata/web/storage.py           |   2 +-
 4 files changed, 128 insertions(+), 96 deletions(-)
 create mode 100644 src/allmydata/web/common_py3.py

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index ee6d7a393..b2327369e 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -18,7 +18,7 @@ from twisted.web.template import flattenString
 # We need to use `nevow.inevow.IRequest` for now for compatibility
 # with the code in web/common.py.  Once nevow bits are gone from
 # web/common.py, we can use `twisted.web.iweb.IRequest` here.
-from nevow.inevow import IRequest
+from twisted.web.iweb import IRequest
 
 from twisted.web.server import Request
 from twisted.web.test.requesthelper import DummyChannel
@@ -36,7 +36,7 @@ from allmydata.web.storage import (
     StorageStatusElement,
     remove_prefix
 )
-from .test_storage import FakeCanary
+from .common_py3 import FakeCanary
 
 def remove_tags(s):
     s = re.sub(r'<[^>]*>', ' ', s)
diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index a930fd2b1..53f78f850 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -15,11 +15,15 @@ from allmydata.interfaces import ExistingChildError, NoSuchChildError, \
      EmptyPathnameComponentError, MustBeDeepImmutableError, \
      MustBeReadonlyError, MustNotBeUnknownRWError, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable.common import UnrecoverableFileError
-from allmydata.util import abbreviate
 from allmydata.util.hashutil import timing_safe_compare
 from allmydata.util.time_format import format_time, format_delta
 from allmydata.util.encodingutil import to_str, quote_output
 
+# Originally part of this module, so still part of its API:
+from .common_py3 import (  # noqa: F401
+    get_arg, abbreviate_time, MultiFormatResource, WebError
+)
+
 
 def get_filenode_metadata(filenode):
     metadata = {'mutable': filenode.is_mutable()}
@@ -104,24 +108,6 @@ def get_root(ctx_or_req):
     link = "/".join([".."] * depth)
     return link
 
-def get_arg(ctx_or_req, argname, default=None, multiple=False):
-    """Extract an argument from either the query args (req.args) or the form
-    body fields (req.fields). If multiple=False, this returns a single value
-    (or the default, which defaults to None), and the query args take
-    precedence. If multiple=True, this returns a tuple of arguments (possibly
-    empty), starting with all those in the query args.
-    """
-    req = IRequest(ctx_or_req)
-    results = []
-    if argname in req.args:
-        results.extend(req.args[argname])
-    if req.fields and argname in req.fields:
-        results.append(req.fields[argname].value)
-    if multiple:
-        return tuple(results)
-    if results:
-        return results[0]
-    return default
 
 def convert_children_json(nodemaker, children_json):
     """I convert the JSON output of GET?t=json into the dict-of-nodes input
@@ -141,20 +127,6 @@ def convert_children_json(nodemaker, children_json):
             children[namex] = (childnode, metadata)
     return children
 
-def abbreviate_time(data):
-    # 1.23s, 790ms, 132us
-    if data is None:
-        return ""
-    s = float(data)
-    if s >= 10:
-        return abbreviate.abbreviate_time(data)
-    if s >= 1.0:
-        return "%.2fs" % s
-    if s >= 0.01:
-        return "%.0fms" % (1000*s)
-    if s >= 0.001:
-        return "%.1fms" % (1000*s)
-    return "%.0fus" % (1000000*s)
 
 def compute_rate(bytes, seconds):
     if bytes is None:
@@ -219,10 +191,6 @@ def render_time(t):
 def render_time_attr(t):
     return format_time(time.localtime(t))
 
-class WebError(Exception):
-    def __init__(self, text, code=http.BAD_REQUEST):
-        self.text = text
-        self.code = code
 
 # XXX: to make UnsupportedMethod return 501 NOT_IMPLEMENTED instead of 500
 # Internal Server Error, we either need to do that ICanHandleException trick,
@@ -421,62 +389,6 @@ class MultiFormatPage(Page):
             return lambda ctx: renderer(IRequest(ctx))
 
 
-class MultiFormatResource(resource.Resource, object):
-    """
-    ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
-    a number of different formats.
-
-    Rendered format is controlled by a query argument (given by
-    ``self.formatArgument``).  Different resources may support different
-    formats but ``json`` is a pretty common one.  ``html`` is the default
-    format if nothing else is given as the ``formatDefault``.
-    """
-    formatArgument = "t"
-    formatDefault = None
-
-    def render(self, req):
-        """
-        Dispatch to a renderer for a particular format, as selected by a query
-        argument.
-
-        A renderer for the format given by the query argument matching
-        ``formatArgument`` will be selected and invoked.  render_HTML will be
-        used as a default if no format is selected (either by query arguments
-        or by ``formatDefault``).
-
-        :return: The result of the selected renderer.
-        """
-        t = get_arg(req, self.formatArgument, self.formatDefault)
-        renderer = self._get_renderer(t)
-        return renderer(req)
-
-    def _get_renderer(self, fmt):
-        """
-        Get the renderer for the indicated format.
-
-        :param str fmt: The format.  If a method with a prefix of ``render_``
-            and a suffix of this format (upper-cased) is found, it will be
-            used.
-
-        :return: A callable which takes a twisted.web Request and renders a
-            response.
-        """
-        renderer = None
-
-        if fmt is not None:
-            try:
-                renderer = getattr(self, "render_{}".format(fmt.upper()))
-            except AttributeError:
-                raise WebError(
-                    "Unknown {} value: {!r}".format(self.formatArgument, fmt),
-                )
-
-        if renderer is None:
-            renderer = self.render_HTML
-
-        return renderer
-
-
 class SlotsSequenceElement(template.Element):
     """
     ``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for
diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py
new file mode 100644
index 000000000..1af8c8892
--- /dev/null
+++ b/src/allmydata/web/common_py3.py
@@ -0,0 +1,120 @@
+"""
+Common utilities that are available from Python 3.
+
+Can eventually be merged back into twisted.web.common.
+"""
+
+from future.utils import PY2
+
+if PY2:
+    from nevow.inevow import IRequest as INevowRequest
+else:
+    INevowRequest = None
+
+from twisted.web import resource, http
+from twisted.web.iweb import IRequest
+
+from allmydata.util import abbreviate
+
+
+class WebError(Exception):
+    def __init__(self, text, code=http.BAD_REQUEST):
+        self.text = text
+        self.code = code
+
+
+def get_arg(ctx_or_req, argname, default=None, multiple=False):
+    """Extract an argument from either the query args (req.args) or the form
+    body fields (req.fields). If multiple=False, this returns a single value
+    (or the default, which defaults to None), and the query args take
+    precedence. If multiple=True, this returns a tuple of arguments (possibly
+    empty), starting with all those in the query args.
+    """
+    results = []
+    if PY2:
+        req = INevowRequest(ctx_or_req)
+        if argname in req.args:
+            results.extend(req.args[argname])
+        if req.fields and argname in req.fields:
+            results.append(req.fields[argname].value)
+    else:
+        req = IRequest(ctx_or_req)
+        if argname in req.args:
+            results.extend(req.args[argname])
+    if multiple:
+        return tuple(results)
+    if results:
+        return results[0]
+    return default
+
+
+class MultiFormatResource(resource.Resource, object):
+    """
+    ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
+    a number of different formats.
+
+    Rendered format is controlled by a query argument (given by
+    ``self.formatArgument``).  Different resources may support different
+    formats but ``json`` is a pretty common one.  ``html`` is the default
+    format if nothing else is given as the ``formatDefault``.
+    """
+    formatArgument = "t"
+    formatDefault = None
+
+    def render(self, req):
+        """
+        Dispatch to a renderer for a particular format, as selected by a query
+        argument.
+
+        A renderer for the format given by the query argument matching
+        ``formatArgument`` will be selected and invoked.  render_HTML will be
+        used as a default if no format is selected (either by query arguments
+        or by ``formatDefault``).
+
+        :return: The result of the selected renderer.
+        """
+        t = get_arg(req, self.formatArgument, self.formatDefault)
+        renderer = self._get_renderer(t)
+        return renderer(req)
+
+    def _get_renderer(self, fmt):
+        """
+        Get the renderer for the indicated format.
+
+        :param str fmt: The format.  If a method with a prefix of ``render_``
+            and a suffix of this format (upper-cased) is found, it will be
+            used.
+
+        :return: A callable which takes a twisted.web Request and renders a
+            response.
+        """
+        renderer = None
+
+        if fmt is not None:
+            try:
+                renderer = getattr(self, "render_{}".format(fmt.upper()))
+            except AttributeError:
+                raise WebError(
+                    "Unknown {} value: {!r}".format(self.formatArgument, fmt),
+                )
+
+        if renderer is None:
+            renderer = self.render_HTML
+
+        return renderer
+
+
+def abbreviate_time(data):
+    # 1.23s, 790ms, 132us
+    if data is None:
+        return ""
+    s = float(data)
+    if s >= 10:
+        return abbreviate.abbreviate_time(data)
+    if s >= 1.0:
+        return "%.2fs" % s
+    if s >= 0.01:
+        return "%.0fms" % (1000*s)
+    if s >= 0.001:
+        return "%.1fms" % (1000*s)
+    return "%.0fus" % (1000000*s)
diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py
index ba6609456..cf3264dac 100644
--- a/src/allmydata/web/storage.py
+++ b/src/allmydata/web/storage.py
@@ -8,7 +8,7 @@ from twisted.web.template import (
     renderer,
     renderElement
 )
-from allmydata.web.common import (
+from allmydata.web.common_py3 import (
     abbreviate_time,
     MultiFormatResource
 )

From 8136b21f464c2ba2a81a73b9c6c3605da4293d46 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:24:21 -0400
Subject: [PATCH 19/39] Skip the tests we aren't porting just yet.

---
 src/allmydata/test/test_storage_web.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index b2327369e..014f41665 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -4,10 +4,13 @@ Tests for twisted.storage that uses Web APIs.
 
 from __future__ import absolute_import
 
+from future.utils import PY2, PY3
+
 import time
 import os.path
 import re
 import json
+from unittest import skipIf
 
 from twisted.trial import unittest
 
@@ -18,7 +21,10 @@ from twisted.web.template import flattenString
 # We need to use `nevow.inevow.IRequest` for now for compatibility
 # with the code in web/common.py.  Once nevow bits are gone from
 # web/common.py, we can use `twisted.web.iweb.IRequest` here.
-from twisted.web.iweb import IRequest
+if PY2:
+    from nevow.inevow import IRequest
+else:
+    from twisted.web.iweb import IRequest
 
 from twisted.web.server import Request
 from twisted.web.test.requesthelper import DummyChannel
@@ -89,8 +95,10 @@ class MyStorageServer(StorageServer):
         self.bucket_counter = MyBucketCountingCrawler(self, statefile)
         self.bucket_counter.setServiceParent(self)
 
+
 class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
+    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -1147,6 +1155,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
 class WebStatus(unittest.TestCase, pollmixin.PollMixin):
 
+    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()

From 5d2bdf58831d88bc27306c1e3d601eb05a42e9a1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:32:41 -0400
Subject: [PATCH 20/39] Explicit bytes.

---
 src/allmydata/storage/mutable.py       | 12 +++---
 src/allmydata/test/test_storage_web.py | 52 +++++++++++++-------------
 2 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py
index bba501693..c108dfe32 100644
--- a/src/allmydata/storage/mutable.py
+++ b/src/allmydata/storage/mutable.py
@@ -87,7 +87,7 @@ class MutableShareFile(object):
                 self.MAGIC, my_nodeid, write_enabler,
                 data_length, extra_lease_offset,
             )
-            leases = ("\x00" * self.LEASE_SIZE) * 4
+            leases = (b"\x00" * self.LEASE_SIZE) * 4
             f.write(header + leases)
             # data goes here, empty after creation
             f.write(struct.pack(">L", num_extra_leases))
@@ -155,7 +155,7 @@ class MutableShareFile(object):
         # Zero out the old lease info (in order to minimize the chance that
         # it could accidentally be exposed to a reader later, re #1528).
         f.seek(old_extra_lease_offset)
-        f.write('\x00' * leases_size)
+        f.write(b'\x00' * leases_size)
         f.flush()
 
         # An interrupt here will corrupt the leases.
@@ -194,7 +194,7 @@ class MutableShareFile(object):
             # Fill any newly exposed empty space with 0's.
             if offset > data_length:
                 f.seek(self.DATA_OFFSET+data_length)
-                f.write('\x00'*(offset - data_length))
+                f.write(b'\x00'*(offset - data_length))
                 f.flush()
 
             new_data_length = offset+length
@@ -326,10 +326,10 @@ class MutableShareFile(object):
         modified = 0
         remaining = 0
         blank_lease = LeaseInfo(owner_num=0,
-                                renew_secret="\x00"*32,
-                                cancel_secret="\x00"*32,
+                                renew_secret=b"\x00"*32,
+                                cancel_secret=b"\x00"*32,
                                 expiration_time=0,
-                                nodeid="\x00"*20)
+                                nodeid=b"\x00"*20)
         with open(self.home, 'rb+') as f:
             for (leasenum,lease) in self._enumerate_leases(f):
                 accepting_nodeids.add(lease.nodeid)
diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index 014f41665..a7dfdf783 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -1,5 +1,7 @@
 """
 Tests for twisted.storage that uses Web APIs.
+
+Partially ported to Python 3.
 """
 
 from __future__ import absolute_import
@@ -283,27 +285,27 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
     def make_shares(self, ss):
         def make(si):
-            return (si, hashutil.tagged_hash("renew", si),
-                    hashutil.tagged_hash("cancel", si))
+            return (si, hashutil.tagged_hash(b"renew", si),
+                    hashutil.tagged_hash(b"cancel", si))
         def make_mutable(si):
-            return (si, hashutil.tagged_hash("renew", si),
-                    hashutil.tagged_hash("cancel", si),
-                    hashutil.tagged_hash("write-enabler", si))
+            return (si, hashutil.tagged_hash(b"renew", si),
+                    hashutil.tagged_hash(b"cancel", si),
+                    hashutil.tagged_hash(b"write-enabler", si))
         def make_extra_lease(si, num):
-            return (hashutil.tagged_hash("renew-%d" % num, si),
-                    hashutil.tagged_hash("cancel-%d" % num, si))
+            return (hashutil.tagged_hash(b"renew-%d" % num, si),
+                    hashutil.tagged_hash(b"cancel-%d" % num, si))
 
-        immutable_si_0, rs0, cs0 = make("\x00" * 16)
-        immutable_si_1, rs1, cs1 = make("\x01" * 16)
+        immutable_si_0, rs0, cs0 = make(b"\x00" * 16)
+        immutable_si_1, rs1, cs1 = make(b"\x01" * 16)
         rs1a, cs1a = make_extra_lease(immutable_si_1, 1)
-        mutable_si_2, rs2, cs2, we2 = make_mutable("\x02" * 16)
-        mutable_si_3, rs3, cs3, we3 = make_mutable("\x03" * 16)
+        mutable_si_2, rs2, cs2, we2 = make_mutable(b"\x02" * 16)
+        mutable_si_3, rs3, cs3, we3 = make_mutable(b"\x03" * 16)
         rs3a, cs3a = make_extra_lease(mutable_si_3, 1)
         sharenums = [0]
         canary = FakeCanary()
         # note: 'tahoe debug dump-share' will not handle this file, since the
         # inner contents are not a valid CHK share
-        data = "\xff" * 1000
+        data = b"\xff" * 1000
 
         a,w = ss.remote_allocate_buckets(immutable_si_0, rs0, cs0, sharenums,
                                          1000, canary)
@@ -330,7 +332,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
     def test_basic(self):
         basedir = "storage/LeaseCrawler/basic"
         fileutil.make_dirs(basedir)
-        ss = InstrumentedStorageServer(basedir, "\x00" * 20)
+        ss = InstrumentedStorageServer(basedir, b"\x00" * 20)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -347,7 +349,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
                           storage_index_to_dir(immutable_si_0),
                           "not-a-share")
         f = open(fn, "wb")
-        f.write("I am not a share.\n")
+        f.write(b"I am not a share.\n")
         f.close()
 
         # this is before the crawl has started, so we're not in a cycle yet
@@ -513,7 +515,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         fileutil.make_dirs(basedir)
         # setting expiration_time to 2000 means that any lease which is more
         # than 2000s old will be expired.
-        ss = InstrumentedStorageServer(basedir, "\x00" * 20,
+        ss = InstrumentedStorageServer(basedir, b"\x00" * 20,
                                        expiration_enabled=True,
                                        expiration_mode="age",
                                        expiration_override_lease_duration=2000)
@@ -653,7 +655,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         # is more than 2000s old will be expired.
         now = time.time()
         then = int(now - 2000)
-        ss = InstrumentedStorageServer(basedir, "\x00" * 20,
+        ss = InstrumentedStorageServer(basedir, b"\x00" * 20,
                                        expiration_enabled=True,
                                        expiration_mode="cutoff-date",
                                        expiration_cutoff_date=then)
@@ -800,7 +802,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         fileutil.make_dirs(basedir)
         now = time.time()
         then = int(now - 2000)
-        ss = StorageServer(basedir, "\x00" * 20,
+        ss = StorageServer(basedir, b"\x00" * 20,
                            expiration_enabled=True,
                            expiration_mode="cutoff-date",
                            expiration_cutoff_date=then,
@@ -857,7 +859,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         fileutil.make_dirs(basedir)
         now = time.time()
         then = int(now - 2000)
-        ss = StorageServer(basedir, "\x00" * 20,
+        ss = StorageServer(basedir, b"\x00" * 20,
                            expiration_enabled=True,
                            expiration_mode="cutoff-date",
                            expiration_cutoff_date=then,
@@ -913,14 +915,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         basedir = "storage/LeaseCrawler/bad_mode"
         fileutil.make_dirs(basedir)
         e = self.failUnlessRaises(ValueError,
-                                  StorageServer, basedir, "\x00" * 20,
+                                  StorageServer, basedir, b"\x00" * 20,
                                   expiration_mode="bogus")
         self.failUnlessIn("GC mode 'bogus' must be 'age' or 'cutoff-date'", str(e))
 
     def test_limited_history(self):
         basedir = "storage/LeaseCrawler/limited_history"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -952,7 +954,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
     def test_unpredictable_future(self):
         basedir = "storage/LeaseCrawler/unpredictable_future"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -1015,7 +1017,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
     def test_no_st_blocks(self):
         basedir = "storage/LeaseCrawler/no_st_blocks"
         fileutil.make_dirs(basedir)
-        ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20,
+        ss = No_ST_BLOCKS_StorageServer(basedir, b"\x00" * 20,
                                         expiration_mode="age",
                                         expiration_override_lease_duration=-1000)
         # a negative expiration_time= means the "configured-"
@@ -1054,7 +1056,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             ]
         basedir = "storage/LeaseCrawler/share_corruption"
         fileutil.make_dirs(basedir)
-        ss = InstrumentedStorageServer(basedir, "\x00" * 20)
+        ss = InstrumentedStorageServer(basedir, b"\x00" * 20)
         w = StorageStatus(ss)
         # make it start sooner than usual.
         lc = ss.lease_checker
@@ -1072,7 +1074,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         fn = os.path.join(ss.sharedir, storage_index_to_dir(first), "0")
         f = open(fn, "rb+")
         f.seek(0)
-        f.write("BAD MAGIC")
+        f.write(b"BAD MAGIC")
         f.close()
         # if get_share_file() doesn't see the correct mutable magic, it
         # assumes the file is an immutable share, and then
@@ -1081,7 +1083,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         # UnknownImmutableContainerVersionError.
 
         # also create an empty bucket
-        empty_si = base32.b2a("\x04"*16)
+        empty_si = base32.b2a(b"\x04"*16)
         empty_bucket_dir = os.path.join(ss.sharedir,
                                         storage_index_to_dir(empty_si))
         fileutil.make_dirs(empty_bucket_dir)

From 0912d5adfc44674d2cd0ccf2fbbaa415f519726d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:43:10 -0400
Subject: [PATCH 21/39] Expirer pass on Python 3.

---
 src/allmydata/test/test_storage_web.py | 116 ++++++++++++++-----------
 1 file changed, 63 insertions(+), 53 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index a7dfdf783..62788cb26 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -47,8 +47,8 @@ from allmydata.web.storage import (
 from .common_py3 import FakeCanary
 
 def remove_tags(s):
-    s = re.sub(r'<[^>]*>', ' ', s)
-    s = re.sub(r'\s+', ' ', s)
+    s = re.sub(br'<[^>]*>', b' ', s)
+    s = re.sub(br'\s+', b' ', s)
     return s
 
 def renderSynchronously(ss):
@@ -408,25 +408,25 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html_in_cycle(html):
             s = remove_tags(html)
-            self.failUnlessIn("So far, this cycle has examined "
-                              "1 shares in 1 buckets (0 mutable / 1 immutable) ", s)
-            self.failUnlessIn("and has recovered: "
-                              "0 shares, 0 buckets (0 mutable / 0 immutable), "
-                              "0 B (0 B / 0 B)", s)
-            self.failUnlessIn("If expiration were enabled, "
-                              "we would have recovered: "
-                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
-                              " 0 B (0 B / 0 B) by now", s)
-            self.failUnlessIn("and the remainder of this cycle "
-                              "would probably recover: "
-                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
-                              " 0 B (0 B / 0 B)", s)
-            self.failUnlessIn("and the whole cycle would probably recover: "
-                              "0 shares, 0 buckets (0 mutable / 0 immutable),"
-                              " 0 B (0 B / 0 B)", s)
-            self.failUnlessIn("if we were strictly using each lease's default "
-                              "31-day lease lifetime", s)
-            self.failUnlessIn("this cycle would be expected to recover: ", s)
+            self.failUnlessIn(b"So far, this cycle has examined "
+                              b"1 shares in 1 buckets (0 mutable / 1 immutable) ", s)
+            self.failUnlessIn(b"and has recovered: "
+                              b"0 shares, 0 buckets (0 mutable / 0 immutable), "
+                              b"0 B (0 B / 0 B)", s)
+            self.failUnlessIn(b"If expiration were enabled, "
+                              b"we would have recovered: "
+                              b"0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              b" 0 B (0 B / 0 B) by now", s)
+            self.failUnlessIn(b"and the remainder of this cycle "
+                              b"would probably recover: "
+                              b"0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              b" 0 B (0 B / 0 B)", s)
+            self.failUnlessIn(b"and the whole cycle would probably recover: "
+                              b"0 shares, 0 buckets (0 mutable / 0 immutable),"
+                              b" 0 B (0 B / 0 B)", s)
+            self.failUnlessIn(b"if we were strictly using each lease's default "
+                              b"31-day lease lifetime", s)
+            self.failUnlessIn(b"this cycle would be expected to recover: ", s)
         d.addCallback(_check_html_in_cycle)
 
         # wait for the crawler to finish the first cycle. Nothing should have
@@ -483,11 +483,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("recovered: 0 shares, 0 buckets "
-                              "(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s)
-            self.failUnlessIn("and saw a total of 4 shares, 4 buckets "
-                              "(2 mutable / 2 immutable),", s)
-            self.failUnlessIn("but expiration was not enabled", s)
+            self.failUnlessIn(b"recovered: 0 shares, 0 buckets "
+                              b"(0 mutable / 0 immutable), 0 B (0 B / 0 B) ", s)
+            self.failUnlessIn(b"and saw a total of 4 shares, 4 buckets "
+                              b"(2 mutable / 2 immutable),", s)
+            self.failUnlessIn(b"but expiration was not enabled", s)
         d.addCallback(_check_html)
         d.addCallback(lambda ign: renderJSON(webstatus))
         def _check_json(raw):
@@ -588,11 +588,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             # predictor thinks we'll have 5 shares and that we'll delete them
             # all. This part of the test depends upon the SIs landing right
             # where they do now.
-            self.failUnlessIn("The remainder of this cycle is expected to "
-                              "recover: 4 shares, 4 buckets", s)
-            self.failUnlessIn("The whole cycle is expected to examine "
-                              "5 shares in 5 buckets and to recover: "
-                              "5 shares, 5 buckets", s)
+            self.failUnlessIn(b"The remainder of this cycle is expected to "
+                              b"recover: 4 shares, 4 buckets", s)
+            self.failUnlessIn(b"The whole cycle is expected to examine "
+                              b"5 shares in 5 buckets and to recover: "
+                              b"5 shares, 5 buckets", s)
         d.addCallback(_check_html_in_cycle)
 
         # wait for the crawler to finish the first cycle. Two shares should
@@ -642,9 +642,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
-            self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s)
-            self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
+            self.failUnlessIn(b"Expiration Enabled: expired leases will be removed", s)
+            self.failUnlessIn(b"Leases created or last renewed more than 33 minutes ago will be considered expired.", s)
+            self.failUnlessIn(b" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
 
@@ -732,11 +732,11 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             # predictor thinks we'll have 5 shares and that we'll delete them
             # all. This part of the test depends upon the SIs landing right
             # where they do now.
-            self.failUnlessIn("The remainder of this cycle is expected to "
-                              "recover: 4 shares, 4 buckets", s)
-            self.failUnlessIn("The whole cycle is expected to examine "
-                              "5 shares in 5 buckets and to recover: "
-                              "5 shares, 5 buckets", s)
+            self.failUnlessIn(b"The remainder of this cycle is expected to "
+                              b"recover: 4 shares, 4 buckets", s)
+            self.failUnlessIn(b"The whole cycle is expected to examine "
+                              b"5 shares in 5 buckets and to recover: "
+                              b"5 shares, 5 buckets", s)
         d.addCallback(_check_html_in_cycle)
 
         # wait for the crawler to finish the first cycle. Two shares should
@@ -788,12 +788,13 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("Expiration Enabled:"
-                              " expired leases will be removed", s)
-            date = time.strftime("%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then))
-            substr = "Leases created or last renewed before %s will be considered expired." % date
+            self.failUnlessIn(b"Expiration Enabled:"
+                              b" expired leases will be removed", s)
+            date = time.strftime(
+                u"%Y-%m-%d (%d-%b-%Y) UTC", time.gmtime(then)).encode("ascii")
+            substr =b"Leases created or last renewed before %s will be considered expired." % date
             self.failUnlessIn(substr, s)
-            self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
+            self.failUnlessIn(b" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
         d.addCallback(_check_html)
         return d
 
@@ -850,7 +851,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("The following sharetypes will be expired: immutable.", s)
+            self.failUnlessIn(b"The following sharetypes will be expired: immutable.", s)
         d.addCallback(_check_html)
         return d
 
@@ -907,7 +908,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         d.addCallback(lambda ign: renderDeferred(webstatus))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("The following sharetypes will be expired: mutable.", s)
+            self.failUnlessIn(b"The following sharetypes will be expired: mutable.", s)
         d.addCallback(_check_html)
         return d
 
@@ -1104,7 +1105,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             rec = so_far["space-recovered"]
             self.failUnlessEqual(rec["examined-buckets"], 1)
             self.failUnlessEqual(rec["examined-shares"], 0)
-            self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)])
+            [(actual_b32, i)] = so_far["corrupt-shares"]
+            actual_b32 = actual_b32.encode("ascii")
+            self.failUnlessEqual((actual_b32, i), (first_b32, 0))
         d.addCallback(_after_first_bucket)
 
         d.addCallback(lambda ign: renderJSON(w))
@@ -1113,13 +1116,16 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             # grr. json turns all dict keys into strings.
             so_far = data["lease-checker"]["cycle-to-date"]
             corrupt_shares = so_far["corrupt-shares"]
-            # it also turns all tuples into lists
-            self.failUnlessEqual(corrupt_shares, [[first_b32, 0]])
+            # it also turns all tuples into lists, and result is unicode (on
+            # Python 3 always, on Python 2 sometimes)
+            [(actual_b32, i)] = corrupt_shares
+            actual_b32 = actual_b32.encode("ascii")
+            self.failUnlessEqual([actual_b32, i], [first_b32, 0])
         d.addCallback(_check_json)
         d.addCallback(lambda ign: renderDeferred(w))
         def _check_html(html):
             s = remove_tags(html)
-            self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s)
+            self.failUnlessIn(b"Corrupt shares: SI %s shnum 0" % first_b32, s)
         d.addCallback(_check_html)
 
         def _wait():
@@ -1132,19 +1138,23 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             rec = last["space-recovered"]
             self.failUnlessEqual(rec["examined-buckets"], 5)
             self.failUnlessEqual(rec["examined-shares"], 3)
-            self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)])
+            [(actual_b32, i)] = last["corrupt-shares"]
+            actual_b32 = actual_b32.encode("ascii")
+            self.failUnlessEqual((actual_b32, i), (first_b32, 0))
         d.addCallback(_after_first_cycle)
         d.addCallback(lambda ign: renderJSON(w))
         def _check_json_history(raw):
             data = json.loads(raw)
             last = data["lease-checker"]["history"]["0"]
             corrupt_shares = last["corrupt-shares"]
-            self.failUnlessEqual(corrupt_shares, [[first_b32, 0]])
+            [(actual_b32, i)] = last["corrupt-shares"]
+            actual_b32 = actual_b32.encode("ascii")
+            self.failUnlessEqual([actual_b32, i], [first_b32, 0])
         d.addCallback(_check_json_history)
         d.addCallback(lambda ign: renderDeferred(w))
         def _check_html_history(html):
             s = remove_tags(html)
-            self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s)
+            self.failUnlessIn(b"Corrupt shares: SI %s shnum 0" % first_b32, s)
         d.addCallback(_check_html_history)
 
         def _cleanup(res):

From b3890a1a45481cc6cea1c1158cf44701b42fd505 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:49:58 -0400
Subject: [PATCH 22/39] Finish porting (expirer-only) tests to Python 3.

---
 src/allmydata/test/test_storage_web.py | 31 ++++++++++++++++----------
 src/allmydata/util/_python3.py         |  1 +
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index 62788cb26..a3c1c7dee 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -5,8 +5,15 @@ Partially ported to Python 3.
 """
 
 from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 from future.utils import PY2, PY3
+if PY2:
+    # Omitted list sinc it broke a test on Python 2. Shouldn't require further
+    # work, when we switch to Python 3 we'll be dropping this, anyway.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, object, range, str, max, min  # noqa: F401
 
 import time
 import os.path
@@ -110,7 +117,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
     def test_bucket_counter(self):
         basedir = "storage/BucketCounter/bucket_counter"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         # to make sure we capture the bucket-counting-crawler in the middle
         # of a cycle, we reach in and reduce its maximum slice time to 0. We
         # also make it start sooner than usual.
@@ -167,7 +174,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
     def test_bucket_counter_cleanup(self):
         basedir = "storage/BucketCounter/bucket_counter_cleanup"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         # to make sure we capture the bucket-counting-crawler in the middle
         # of a cycle, we reach in and reduce its maximum slice time to 0.
         ss.bucket_counter.slow_start = 0
@@ -200,16 +207,16 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
         def _check2(ignored):
             ss.bucket_counter.cpu_slice = orig_cpu_slice
             s = ss.bucket_counter.get_state()
-            self.failIf(-12 in s["bucket-counts"], s["bucket-counts"].keys())
+            self.failIf(-12 in s["bucket-counts"], list(s["bucket-counts"].keys()))
             self.failIf("bogusprefix!" in s["storage-index-samples"],
-                        s["storage-index-samples"].keys())
+                        list(s["storage-index-samples"].keys()))
         d.addCallback(_check2)
         return d
 
     def test_bucket_counter_eta(self):
         basedir = "storage/BucketCounter/bucket_counter_eta"
         fileutil.make_dirs(basedir)
-        ss = MyStorageServer(basedir, "\x00" * 20)
+        ss = MyStorageServer(basedir, b"\x00" * 20)
         ss.bucket_counter.slow_start = 0
         # these will be fired inside finished_prefix()
         hooks = ss.bucket_counter.hook_ds = [defer.Deferred() for i in range(3)]
@@ -1182,7 +1189,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_status(self):
         basedir = "storage/WebStatus/status"
         fileutil.make_dirs(basedir)
-        nodeid = "\x00" * 20
+        nodeid = b"\x00" * 20
         ss = StorageServer(basedir, nodeid)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss, "nickname")
@@ -1216,7 +1223,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         # (test runs on all platforms).
         basedir = "storage/WebStatus/status_no_disk_stats"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
@@ -1236,7 +1243,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         # show that no shares will be accepted, and get_available_space() should be 0.
         basedir = "storage/WebStatus/status_bad_disk_stats"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20)
+        ss = StorageServer(basedir, b"\x00" * 20)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
@@ -1256,7 +1263,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
 
         basedir = "storage/WebStatus/status_right_disk_stats"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20, reserved_space=reserved)
+        ss = StorageServer(basedir, b"\x00" * 20, reserved_space=reserved)
         expecteddir = ss.sharedir
 
         def call_get_disk_stats(whichdir, reserved_space=0):
@@ -1290,7 +1297,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_readonly(self):
         basedir = "storage/WebStatus/readonly"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True)
+        ss = StorageServer(basedir, b"\x00" * 20, readonly_storage=True)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
@@ -1301,7 +1308,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_reserved(self):
         basedir = "storage/WebStatus/reserved"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6)
+        ss = StorageServer(basedir, b"\x00" * 20, reserved_space=10e6)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
@@ -1312,7 +1319,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_huge_reserved(self):
         basedir = "storage/WebStatus/reserved"
         fileutil.make_dirs(basedir)
-        ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6)
+        ss = StorageServer(basedir, b"\x00" * 20, reserved_space=10e6)
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 794edef40..b54d0a28f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -81,6 +81,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
+    "allmydata.test.test_storage_web",  # partial, WIP
     "allmydata.test.test_time_format",
     "allmydata.test.test_util",
     "allmydata.test.test_version",

From 6fd8ae1cc924b8d5f24547466e14f7c9916f2376 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 20 Aug 2020 14:55:49 -0400
Subject: [PATCH 23/39] Finish port to Python 3.

---
 src/allmydata/storage/expirer.py | 10 ++++++++++
 src/allmydata/util/_python3.py   |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/storage/expirer.py b/src/allmydata/storage/expirer.py
index a13c188bd..ffe2bf774 100644
--- a/src/allmydata/storage/expirer.py
+++ b/src/allmydata/storage/expirer.py
@@ -1,3 +1,13 @@
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # We omit anything that might end up in pickle, just in case.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min  # noqa: F401
+
 import time, os, pickle, struct
 from allmydata.storage.crawler import ShareCrawler
 from allmydata.storage.shares import get_share_file
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index b54d0a28f..da9295f4e 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -33,6 +33,7 @@ PORTED_MODULES = [
     "allmydata.hashtree",
     "allmydata.immutable.happiness_upload",
     "allmydata.storage.crawler",
+    "allmydata.storage.expirer",
     "allmydata.test.common_py3",
     "allmydata.util._python3",
     "allmydata.util.abbreviate",

From 625e2611c11796f84ac4f83a527bf847a8f49f89 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 24 Aug 2020 11:59:52 -0400
Subject: [PATCH 24/39] Address some review comments.

---
 src/allmydata/test/test_storage_web.py | 4 +---
 src/allmydata/web/common_py3.py        | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index a3c1c7dee..9c64c2f45 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -1123,8 +1123,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             # grr. json turns all dict keys into strings.
             so_far = data["lease-checker"]["cycle-to-date"]
             corrupt_shares = so_far["corrupt-shares"]
-            # it also turns all tuples into lists, and result is unicode (on
-            # Python 3 always, on Python 2 sometimes)
+            # it also turns all tuples into lists, and result is unicode:
             [(actual_b32, i)] = corrupt_shares
             actual_b32 = actual_b32.encode("ascii")
             self.failUnlessEqual([actual_b32, i], [first_b32, 0])
@@ -1153,7 +1152,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
         def _check_json_history(raw):
             data = json.loads(raw)
             last = data["lease-checker"]["history"]["0"]
-            corrupt_shares = last["corrupt-shares"]
             [(actual_b32, i)] = last["corrupt-shares"]
             actual_b32 = actual_b32.encode("ascii")
             self.failUnlessEqual([actual_b32, i], [first_b32, 0])
diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py
index 1af8c8892..06751a8e8 100644
--- a/src/allmydata/web/common_py3.py
+++ b/src/allmydata/web/common_py3.py
@@ -1,7 +1,7 @@
 """
 Common utilities that are available from Python 3.
 
-Can eventually be merged back into twisted.web.common.
+Can eventually be merged back into allmydata.web.common.
 """
 
 from future.utils import PY2

From e22bed447bc4b3ccaf29b998fb8a8a72a418bef0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 24 Aug 2020 13:10:17 -0400
Subject: [PATCH 25/39] Some updates for URI tests.

---
 src/allmydata/test/test_uri.py | 87 +++++++++++++++++++++-------------
 src/allmydata/util/_python3.py |  1 +
 2 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py
index c04b1259d..a14e0c304 100644
--- a/src/allmydata/test/test_uri.py
+++ b/src/allmydata/test/test_uri.py
@@ -1,4 +1,17 @@
 
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min  # noqa: F401
+
 import os
 from twisted.trial import unittest
 from allmydata import uri
@@ -40,24 +53,24 @@ class Literal(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(u.get_verify_cap(), None)
 
     def test_empty(self):
-        data = "" # This data is some *very* small data!
+        data = b"" # This data is some *very* small data!
         return self._help_test(data)
 
     def test_pack(self):
-        data = "This is some small data"
+        data = b"This is some small data"
         return self._help_test(data)
 
     def test_nonascii(self):
-        data = "This contains \x00 and URI:LIT: and \n, oh my."
+        data = b"This contains \x00 and URI:LIT: and \n, oh my."
         return self._help_test(data)
 
 class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_compare(self):
-        lit1 = uri.LiteralFileURI("some data")
+        lit1 = uri.LiteralFileURI(b"some data")
         fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834'
         chk1 = uri.CHKFileURI.init_from_string(fileURI)
         chk2 = uri.CHKFileURI.init_from_string(fileURI)
-        unk = uri.UnknownURI("lafs://from_the_future")
+        unk = uri.UnknownURI(b"lafs://from_the_future")
         self.failIfEqual(lit1, chk1)
         self.failUnlessReallyEqual(chk1, chk2)
         self.failIfEqual(chk1, "not actually a URI")
@@ -66,12 +79,12 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(len(s), 3) # since chk1==chk2
 
     def test_is_uri(self):
-        lit1 = uri.LiteralFileURI("some data").to_string()
+        lit1 = uri.LiteralFileURI(b"some data").to_string()
         self.failUnless(uri.is_uri(lit1))
         self.failIf(uri.is_uri(None))
 
     def test_is_literal_file_uri(self):
-        lit1 = uri.LiteralFileURI("some data").to_string()
+        lit1 = uri.LiteralFileURI(b"some data").to_string()
         self.failUnless(uri.is_literal_file_uri(lit1))
         self.failIf(uri.is_literal_file_uri(None))
         self.failIf(uri.is_literal_file_uri("foo"))
@@ -89,9 +102,9 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_pack(self):
-        key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+        key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
         storage_index = hashutil.storage_index_hash(key)
-        uri_extension_hash = hashutil.uri_extension_hash("stuff")
+        uri_extension_hash = hashutil.uri_extension_hash(b"stuff")
         needed_shares = 25
         total_shares = 100
         size = 1234
@@ -145,8 +158,8 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
         v2 = uri.from_string(v.to_string())
         self.failUnlessReallyEqual(v, v2)
 
-        v3 = uri.CHKFileVerifierURI(storage_index="\x00"*16,
-                                    uri_extension_hash="\x00"*32,
+        v3 = uri.CHKFileVerifierURI(storage_index=b"\x00"*16,
+                                    uri_extension_hash=b"\x00"*32,
                                     needed_shares=3,
                                     total_shares=10,
                                     size=1234)
@@ -155,9 +168,9 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failIf(v3.is_mutable())
 
     def test_pack_badly(self):
-        key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+        key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
         storage_index = hashutil.storage_index_hash(key)
-        uri_extension_hash = hashutil.uri_extension_hash("stuff")
+        uri_extension_hash = hashutil.uri_extension_hash(b"stuff")
         needed_shares = 25
         total_shares = 100
         size = 1234
@@ -186,23 +199,23 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class Extension(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_pack(self):
-        data = {"stuff": "value",
+        data = {"stuff": b"value",
                 "size": 12,
                 "needed_shares": 3,
-                "big_hash": hashutil.tagged_hash("foo", "bar"),
+                "big_hash": hashutil.tagged_hash(b"foo", b"bar"),
                 }
         ext = uri.pack_extension(data)
         d = uri.unpack_extension(ext)
-        self.failUnlessReallyEqual(d["stuff"], "value")
+        self.failUnlessReallyEqual(d["stuff"], b"value")
         self.failUnlessReallyEqual(d["size"], 12)
-        self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash("foo", "bar"))
+        self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash(b"foo", b"bar"))
 
         readable = uri.unpack_extension_readable(ext)
         self.failUnlessReallyEqual(readable["needed_shares"], 3)
-        self.failUnlessReallyEqual(readable["stuff"], "value")
+        self.failUnlessReallyEqual(readable["stuff"], b"value")
         self.failUnlessReallyEqual(readable["size"], 12)
         self.failUnlessReallyEqual(readable["big_hash"],
-                             base32.b2a(hashutil.tagged_hash("foo", "bar")))
+                             base32.b2a(hashutil.tagged_hash(b"foo", b"bar")))
         self.failUnlessReallyEqual(readable["UEB_hash"],
                              base32.b2a(hashutil.uri_extension_hash(ext)))
 
@@ -222,7 +235,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(isinstance(u2.get_error(), CapConstraintError))
 
         # Future caps might have non-ASCII chars in them. (Or maybe not, who can tell about the future?)
-        future_uri = u"I am a cap from the \u263A future. Whatever you ".encode('utf-8')
+        future_uri = u"I am a cap from the \u263A future. Whatever you "
         u = uri.from_string(future_uri)
         self.failUnless(isinstance(u, uri.UnknownURI))
         self.failUnlessReallyEqual(u.to_string(), future_uri)
@@ -243,8 +256,8 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
     def setUp(self):
-        self.writekey = "\x01" * 16
-        self.fingerprint = "\x02" * 32
+        self.writekey = b"\x01" * 16
+        self.fingerprint = b"\x02" * 32
         self.readkey = hashutil.ssk_readkey_hash(self.writekey)
         self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
 
@@ -417,7 +430,13 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failIf(u2.is_readonly())
         self.failUnless(u2.is_mutable())
 
-        cap3 = cap+":"+os.urandom(40) # parse *that*!
+
+        cap3 = cap+":"
+        for item in os.urandom(40):
+            if isinstance(item, int):
+                cap3 += chr(item)
+            else:
+                cap3 += chr(ord(item))
         u3 = uri.WriteableMDMFFileURI.init_from_string(cap3)
         self.failUnlessReallyEqual(self.writekey, u3.writekey)
         self.failUnlessReallyEqual(self.fingerprint, u3.fingerprint)
@@ -468,8 +487,8 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_pack(self):
-        writekey = "\x01" * 16
-        fingerprint = "\x02" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\x02" * 32
 
         n = uri.WriteableSSKFileURI(writekey, fingerprint)
         u1 = uri.DirectoryURI(n)
@@ -536,8 +555,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
                                  u1.get_verify_cap()._filenode_uri)
 
     def test_immutable(self):
-        readkey = "\x01" * 16
-        uri_extension_hash = hashutil.uri_extension_hash("stuff")
+        readkey = b"\x01" * 16
+        uri_extension_hash = hashutil.uri_extension_hash(b"stuff")
         needed_shares = 3
         total_shares = 10
         size = 1234
@@ -597,7 +616,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(str(u2_verifier))
 
     def test_literal(self):
-        u0 = uri.LiteralFileURI("data")
+        u0 = uri.LiteralFileURI(b"data")
         u1 = uri.LiteralDirectoryURI(u0)
         self.failUnless(str(u1))
         self.failUnlessReallyEqual(u1.to_string(), "URI:DIR2-LIT:mrqxiyi")
@@ -611,8 +630,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(u1.abbrev_si(), "<LIT>")
 
     def test_mdmf(self):
-        writekey = "\x01" * 16
-        fingerprint = "\x02" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\x02" * 32
         uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
         d1 = uri.MDMFDirectoryURI(uri1)
         self.failIf(d1.is_readonly())
@@ -635,8 +654,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessIsInstance(d3, uri.UnknownURI)
 
     def test_mdmf_attenuation(self):
-        writekey = "\x01" * 16
-        fingerprint = "\x02" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\x02" * 32
 
         uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
         d1 = uri.MDMFDirectoryURI(uri1)
@@ -676,8 +695,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
 
     def test_mdmf_verifier(self):
         # I'm not sure what I want to write here yet.
-        writekey = "\x01" * 16
-        fingerprint = "\x02" * 32
+        writekey = b"\x01" * 16
+        fingerprint = b"\x02" * 32
         uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
         d1 = uri.MDMFDirectoryURI(uri1)
         v1 = d1.get_verify_cap()
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 794edef40..84c08a36c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -82,6 +82,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
     "allmydata.test.test_time_format",
+    "allmydata.test.test_uri",
     "allmydata.test.test_util",
     "allmydata.test.test_version",
 ]

From 388f27d2066a2611e353fe5a4786a89daf778495 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 24 Aug 2020 13:41:58 -0400
Subject: [PATCH 26/39] Some more porting, tests pass on Python 2.

---
 src/allmydata/test/test_uri.py |  42 +++----
 src/allmydata/uri.py           | 221 +++++++++++++++++----------------
 2 files changed, 137 insertions(+), 126 deletions(-)

diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py
index a14e0c304..4b9c229f0 100644
--- a/src/allmydata/test/test_uri.py
+++ b/src/allmydata/test/test_uri.py
@@ -67,7 +67,7 @@ class Literal(testutil.ReallyEqualMixin, unittest.TestCase):
 class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_compare(self):
         lit1 = uri.LiteralFileURI(b"some data")
-        fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834'
+        fileURI = b'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834'
         chk1 = uri.CHKFileURI.init_from_string(fileURI)
         chk2 = uri.CHKFileURI.init_from_string(fileURI)
         unk = uri.UnknownURI(b"lafs://from_the_future")
@@ -89,11 +89,14 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failIf(uri.is_literal_file_uri(None))
         self.failIf(uri.is_literal_file_uri("foo"))
         self.failIf(uri.is_literal_file_uri("ro.foo"))
-        self.failIf(uri.is_literal_file_uri("URI:LITfoo"))
+        self.failIf(uri.is_literal_file_uri(b"URI:LITfoo"))
         self.failUnless(uri.is_literal_file_uri("ro.URI:LIT:foo"))
         self.failUnless(uri.is_literal_file_uri("imm.URI:LIT:foo"))
 
     def test_has_uri_prefix(self):
+        self.failUnless(uri.has_uri_prefix(b"URI:foo"))
+        self.failUnless(uri.has_uri_prefix(b"ro.URI:foo"))
+        self.failUnless(uri.has_uri_prefix(b"imm.URI:foo"))
         self.failUnless(uri.has_uri_prefix("URI:foo"))
         self.failUnless(uri.has_uri_prefix("ro.URI:foo"))
         self.failUnless(uri.has_uri_prefix("imm.URI:foo"))
@@ -151,7 +154,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(u.to_string(), u2imm.to_string())
 
         v = u.get_verify_cap()
-        self.failUnless(isinstance(v.to_string(), str))
+        self.failUnless(isinstance(v.to_string(), bytes))
         self.failUnless(v.is_readonly())
         self.failIf(v.is_mutable())
 
@@ -163,7 +166,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
                                     needed_shares=3,
                                     total_shares=10,
                                     size=1234)
-        self.failUnless(isinstance(v3.to_string(), str))
+        self.failUnless(isinstance(v3.to_string(), bytes))
         self.failUnless(v3.is_readonly())
         self.failIf(v3.is_mutable())
 
@@ -222,12 +225,14 @@ class Extension(testutil.ReallyEqualMixin, unittest.TestCase):
 class Unknown(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_from_future(self):
         # any URI type that we don't recognize should be treated as unknown
-        future_uri = "I am a URI from the future. Whatever you do, don't "
+        future_uri = b"I am a URI from the future. Whatever you do, don't "
         u = uri.from_string(future_uri)
         self.failUnless(isinstance(u, uri.UnknownURI))
         self.failUnlessReallyEqual(u.to_string(), future_uri)
         self.failUnless(u.get_readonly() is None)
         self.failUnless(u.get_error() is None)
+        future_uri_unicode = future_uri.decode("utf-8")
+        self.assertEqual(future_uri, uri.from_string(future_uri_unicode).to_string())
 
         u2 = uri.UnknownURI(future_uri, error=CapConstraintError("..."))
         self.failUnlessReallyEqual(u.to_string(), future_uri)
@@ -235,7 +240,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(isinstance(u2.get_error(), CapConstraintError))
 
         # Future caps might have non-ASCII chars in them. (Or maybe not, who can tell about the future?)
-        future_uri = u"I am a cap from the \u263A future. Whatever you "
+        future_uri = u"I am a cap from the \u263A future. Whatever you ".encode("utf-8")
         u = uri.from_string(future_uri)
         self.failUnless(isinstance(u, uri.UnknownURI))
         self.failUnlessReallyEqual(u.to_string(), future_uri)
@@ -251,7 +256,7 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_constraint(self):
         bad = "http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/"
         self.failUnlessRaises(uri.BadURIError, uri.DirectoryURI.init_from_string, bad)
-        fileURI = 'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834'
+        fileURI = b'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834'
         uri.CHKFileURI.init_from_string(fileURI)
 
 class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
@@ -423,7 +428,7 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
         u1 = uri.WriteableMDMFFileURI(self.writekey, self.fingerprint)
         cap = u1.to_string()
 
-        cap2 = cap+":I COME FROM THE FUTURE"
+        cap2 = cap+b":I COME FROM THE FUTURE"
         u2 = uri.WriteableMDMFFileURI.init_from_string(cap2)
         self.failUnlessReallyEqual(self.writekey, u2.writekey)
         self.failUnlessReallyEqual(self.fingerprint, u2.fingerprint)
@@ -431,26 +436,21 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(u2.is_mutable())
 
 
-        cap3 = cap+":"
-        for item in os.urandom(40):
-            if isinstance(item, int):
-                cap3 += chr(item)
-            else:
-                cap3 += chr(ord(item))
+        cap3 = cap+b":" + os.urandom(40)
         u3 = uri.WriteableMDMFFileURI.init_from_string(cap3)
         self.failUnlessReallyEqual(self.writekey, u3.writekey)
         self.failUnlessReallyEqual(self.fingerprint, u3.fingerprint)
         self.failIf(u3.is_readonly())
         self.failUnless(u3.is_mutable())
 
-        cap4 = u1.get_readonly().to_string()+":ooh scary future stuff"
+        cap4 = u1.get_readonly().to_string()+b":ooh scary future stuff"
         u4 = uri.from_string_mutable_filenode(cap4)
         self.failUnlessReallyEqual(self.readkey, u4.readkey)
         self.failUnlessReallyEqual(self.fingerprint, u4.fingerprint)
         self.failUnless(u4.is_readonly())
         self.failUnless(u4.is_mutable())
 
-        cap5 = u1.get_verify_cap().to_string()+":spoilers!"
+        cap5 = u1.get_verify_cap().to_string()+b":spoilers!"
         u5 = uri.from_string(cap5)
         self.failUnlessReallyEqual(self.storage_index, u5.storage_index)
         self.failUnlessReallyEqual(self.fingerprint, u5.fingerprint)
@@ -567,7 +567,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
                                total_shares=total_shares,
                                size=size)
         fncap = fnuri.to_string()
-        self.failUnlessReallyEqual(fncap, "URI:CHK:aeaqcaibaeaqcaibaeaqcaibae:nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa:3:10:1234")
+        self.failUnlessReallyEqual(fncap, b"URI:CHK:aeaqcaibaeaqcaibaeaqcaibae:nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa:3:10:1234")
         u1 = uri.ImmutableDirectoryURI(fnuri)
         self.failUnless(u1.is_readonly())
         self.failIf(u1.is_mutable())
@@ -606,20 +606,20 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(IVerifierURI.providedBy(u2_verifier))
         u2vs = u2_verifier.to_string()
         # URI:DIR2-CHK-Verifier:$key:$ueb:$k:$n:$size
-        self.failUnless(u2vs.startswith("URI:DIR2-CHK-Verifier:"), u2vs)
+        self.failUnless(u2vs.startswith(b"URI:DIR2-CHK-Verifier:"), u2vs)
         u2_verifier_fileuri = u2_verifier.get_filenode_cap()
         self.failUnless(IVerifierURI.providedBy(u2_verifier_fileuri))
         u2vfs = u2_verifier_fileuri.to_string()
         # URI:CHK-Verifier:$key:$ueb:$k:$n:$size
         self.failUnlessReallyEqual(u2vfs, fnuri.get_verify_cap().to_string())
-        self.failUnlessReallyEqual(u2vs[len("URI:DIR2-"):], u2vfs[len("URI:"):])
+        self.failUnlessReallyEqual(u2vs[len(b"URI:DIR2-"):], u2vfs[len(b"URI:"):])
         self.failUnless(str(u2_verifier))
 
     def test_literal(self):
         u0 = uri.LiteralFileURI(b"data")
         u1 = uri.LiteralDirectoryURI(u0)
         self.failUnless(str(u1))
-        self.failUnlessReallyEqual(u1.to_string(), "URI:DIR2-LIT:mrqxiyi")
+        self.failUnlessReallyEqual(u1.to_string(), b"URI:DIR2-LIT:mrqxiyi")
         self.failUnless(u1.is_readonly())
         self.failIf(u1.is_mutable())
         self.failUnless(IURI.providedBy(u1))
@@ -627,7 +627,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnless(IDirnodeURI.providedBy(u1))
         self.failUnlessReallyEqual(u1.get_verify_cap(), None)
         self.failUnlessReallyEqual(u1.get_storage_index(), None)
-        self.failUnlessReallyEqual(u1.abbrev_si(), "<LIT>")
+        self.failUnlessReallyEqual(u1.abbrev_si(), b"<LIT>")
 
     def test_mdmf(self):
         writekey = b"\x01" * 16
diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index 051b45f79..70fd80c92 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -1,3 +1,4 @@
+from past.builtins import unicode, long
 
 import re
 
@@ -24,10 +25,10 @@ class BadURIError(CapConstraintError):
 #  - make variable and method names consistently use _uri for an URI string,
 #    and _cap for a Cap object (decoded URI)
 
-BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits)
-BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits)
+BASE32STR_128bits = b'(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits)
+BASE32STR_256bits = b'(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits)
 
-NUMBER='([0-9]+)'
+NUMBER=b'([0-9]+)'
 
 
 class _BaseURI(object):
@@ -53,10 +54,10 @@ class _BaseURI(object):
 @implementer(IURI, IImmutableFileURI)
 class CHKFileURI(_BaseURI):
 
-    BASE_STRING='URI:CHK:'
-    STRING_RE=re.compile('^URI:CHK:'+BASE32STR_128bits+':'+
-                         BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER+
-                         '$')
+    BASE_STRING=b'URI:CHK:'
+    STRING_RE=re.compile(b'^URI:CHK:'+BASE32STR_128bits+b':'+
+                         BASE32STR_256bits+b':'+NUMBER+b':'+NUMBER+b':'+NUMBER+
+                         b'$')
 
     def __init__(self, key, uri_extension_hash, needed_shares, total_shares,
                  size):
@@ -82,7 +83,7 @@ class CHKFileURI(_BaseURI):
         assert isinstance(self.total_shares, int)
         assert isinstance(self.size, (int,long))
 
-        return ('URI:CHK:%s:%s:%d:%d:%d' %
+        return (b'URI:CHK:%s:%s:%d:%d:%d' %
                 (base32.b2a(self.key),
                  base32.b2a(self.uri_extension_hash),
                  self.needed_shares,
@@ -112,9 +113,9 @@ class CHKFileURI(_BaseURI):
 @implementer(IVerifierURI)
 class CHKFileVerifierURI(_BaseURI):
 
-    BASE_STRING='URI:CHK-Verifier:'
-    STRING_RE=re.compile('^URI:CHK-Verifier:'+BASE32STR_128bits+':'+
-                         BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER)
+    BASE_STRING=b'URI:CHK-Verifier:'
+    STRING_RE=re.compile(b'^URI:CHK-Verifier:'+BASE32STR_128bits+b':'+
+                         BASE32STR_256bits+b':'+NUMBER+b':'+NUMBER+b':'+NUMBER)
 
     def __init__(self, storage_index, uri_extension_hash,
                  needed_shares, total_shares, size):
@@ -138,7 +139,7 @@ class CHKFileVerifierURI(_BaseURI):
         assert isinstance(self.total_shares, int)
         assert isinstance(self.size, (int,long))
 
-        return ('URI:CHK-Verifier:%s:%s:%d:%d:%d' %
+        return (b'URI:CHK-Verifier:%s:%s:%d:%d:%d' %
                 (si_b2a(self.storage_index),
                  base32.b2a(self.uri_extension_hash),
                  self.needed_shares,
@@ -161,8 +162,8 @@ class CHKFileVerifierURI(_BaseURI):
 @implementer(IURI, IImmutableFileURI)
 class LiteralFileURI(_BaseURI):
 
-    BASE_STRING='URI:LIT:'
-    STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$')
+    BASE_STRING=b'URI:LIT:'
+    STRING_RE=re.compile(b'^URI:LIT:'+base32.BASE32STR_anybytes+b'$')
 
     def __init__(self, data=None):
         if data is not None:
@@ -177,7 +178,7 @@ class LiteralFileURI(_BaseURI):
         return cls(base32.a2b(mo.group(1)))
 
     def to_string(self):
-        return 'URI:LIT:%s' % base32.b2a(self.data)
+        return b'URI:LIT:%s' % base32.b2a(self.data)
 
     def is_readonly(self):
         return True
@@ -202,9 +203,9 @@ class LiteralFileURI(_BaseURI):
 @implementer(IURI, IMutableFileURI)
 class WriteableSSKFileURI(_BaseURI):
 
-    BASE_STRING='URI:SSK:'
-    STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+
-                         BASE32STR_256bits+'$')
+    BASE_STRING=b'URI:SSK:'
+    STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+
+                         BASE32STR_256bits+b'$')
 
     def __init__(self, writekey, fingerprint):
         self.writekey = writekey
@@ -223,8 +224,8 @@ class WriteableSSKFileURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.writekey, str)
         assert isinstance(self.fingerprint, str)
-        return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey),
-                                  base32.b2a(self.fingerprint))
+        return b'URI:SSK:%s:%s' % (base32.b2a(self.writekey),
+                                   base32.b2a(self.fingerprint))
 
     def __repr__(self):
         return "<%s %s>" % (self.__class__.__name__, self.abbrev())
@@ -251,8 +252,8 @@ class WriteableSSKFileURI(_BaseURI):
 @implementer(IURI, IMutableFileURI)
 class ReadonlySSKFileURI(_BaseURI):
 
-    BASE_STRING='URI:SSK-RO:'
-    STRING_RE=re.compile('^URI:SSK-RO:'+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
+    BASE_STRING=b'URI:SSK-RO:'
+    STRING_RE=re.compile(b'^URI:SSK-RO:'+BASE32STR_128bits+b':'+BASE32STR_256bits+b'$')
 
     def __init__(self, readkey, fingerprint):
         self.readkey = readkey
@@ -270,8 +271,8 @@ class ReadonlySSKFileURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.readkey, str)
         assert isinstance(self.fingerprint, str)
-        return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey),
-                                     base32.b2a(self.fingerprint))
+        return b'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey),
+                                      base32.b2a(self.fingerprint))
 
     def __repr__(self):
         return "<%s %s>" % (self.__class__.__name__, self.abbrev())
@@ -298,8 +299,8 @@ class ReadonlySSKFileURI(_BaseURI):
 @implementer(IVerifierURI)
 class SSKVerifierURI(_BaseURI):
 
-    BASE_STRING='URI:SSK-Verifier:'
-    STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'$')
+    BASE_STRING=b'URI:SSK-Verifier:'
+    STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'$')
 
     def __init__(self, storage_index, fingerprint):
         assert len(storage_index) == 16
@@ -316,8 +317,8 @@ class SSKVerifierURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.storage_index, str)
         assert isinstance(self.fingerprint, str)
-        return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index),
-                                           base32.b2a(self.fingerprint))
+        return b'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index),
+                                            base32.b2a(self.fingerprint))
 
     def is_readonly(self):
         return True
@@ -335,8 +336,8 @@ class SSKVerifierURI(_BaseURI):
 @implementer(IURI, IMutableFileURI)
 class WriteableMDMFFileURI(_BaseURI):
 
-    BASE_STRING='URI:MDMF:'
-    STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)')
+    BASE_STRING=b'URI:MDMF:'
+    STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)')
 
     def __init__(self, writekey, fingerprint):
         self.writekey = writekey
@@ -355,8 +356,8 @@ class WriteableMDMFFileURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.writekey, str)
         assert isinstance(self.fingerprint, str)
-        ret = 'URI:MDMF:%s:%s' % (base32.b2a(self.writekey),
-                                  base32.b2a(self.fingerprint))
+        ret = b'URI:MDMF:%s:%s' % (base32.b2a(self.writekey),
+                                   base32.b2a(self.fingerprint))
         return ret
 
     def __repr__(self):
@@ -384,8 +385,8 @@ class WriteableMDMFFileURI(_BaseURI):
 @implementer(IURI, IMutableFileURI)
 class ReadonlyMDMFFileURI(_BaseURI):
 
-    BASE_STRING='URI:MDMF-RO:'
-    STRING_RE=re.compile('^' +BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)')
+    BASE_STRING=b'URI:MDMF-RO:'
+    STRING_RE=re.compile(b'^' +BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)')
 
     def __init__(self, readkey, fingerprint):
         self.readkey = readkey
@@ -404,8 +405,8 @@ class ReadonlyMDMFFileURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.readkey, str)
         assert isinstance(self.fingerprint, str)
-        ret = 'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey),
-                                     base32.b2a(self.fingerprint))
+        ret = b'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey),
+                                      base32.b2a(self.fingerprint))
         return ret
 
     def __repr__(self):
@@ -433,8 +434,8 @@ class ReadonlyMDMFFileURI(_BaseURI):
 @implementer(IVerifierURI)
 class MDMFVerifierURI(_BaseURI):
 
-    BASE_STRING='URI:MDMF-Verifier:'
-    STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)')
+    BASE_STRING=b'URI:MDMF-Verifier:'
+    STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)')
 
     def __init__(self, storage_index, fingerprint):
         assert len(storage_index) == 16
@@ -451,8 +452,8 @@ class MDMFVerifierURI(_BaseURI):
     def to_string(self):
         assert isinstance(self.storage_index, str)
         assert isinstance(self.fingerprint, str)
-        ret = 'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index),
-                                           base32.b2a(self.fingerprint))
+        ret = b'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index),
+                                            base32.b2a(self.fingerprint))
         return ret
 
     def is_readonly(self):
@@ -494,12 +495,12 @@ class _DirectoryBaseURI(_BaseURI):
         return self.BASE_STRING+bits
 
     def abbrev(self):
-        return self._filenode_uri.to_string().split(':')[2][:5]
+        return self._filenode_uri.to_string().split(b':')[2][:5]
 
     def abbrev_si(self):
         si = self._filenode_uri.get_storage_index()
         if si is None:
-            return "<LIT>"
+            return b"<LIT>"
         return base32.b2a(si)[:5]
 
     def is_mutable(self):
@@ -518,8 +519,8 @@ class _DirectoryBaseURI(_BaseURI):
 @implementer(IDirectoryURI)
 class DirectoryURI(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=WriteableSSKFileURI
 
     def __init__(self, filenode_uri=None):
@@ -537,8 +538,8 @@ class DirectoryURI(_DirectoryBaseURI):
 @implementer(IReadonlyDirectoryURI)
 class ReadonlyDirectoryURI(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2-RO:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-RO:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=ReadonlySSKFileURI
 
     def __init__(self, filenode_uri=None):
@@ -571,8 +572,8 @@ class _ImmutableDirectoryBaseURI(_DirectoryBaseURI):
 
 
 class ImmutableDirectoryURI(_ImmutableDirectoryBaseURI):
-    BASE_STRING='URI:DIR2-CHK:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-CHK:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=CHKFileURI
 
     def get_verify_cap(self):
@@ -581,8 +582,8 @@ class ImmutableDirectoryURI(_ImmutableDirectoryBaseURI):
 
 
 class LiteralDirectoryURI(_ImmutableDirectoryBaseURI):
-    BASE_STRING='URI:DIR2-LIT:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-LIT:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=LiteralFileURI
 
     def get_verify_cap(self):
@@ -593,8 +594,8 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI):
 @implementer(IDirectoryURI)
 class MDMFDirectoryURI(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2-MDMF:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-MDMF:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=WriteableMDMFFileURI
 
     def __init__(self, filenode_uri=None):
@@ -615,8 +616,8 @@ class MDMFDirectoryURI(_DirectoryBaseURI):
 @implementer(IReadonlyDirectoryURI)
 class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2-MDMF-RO:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-MDMF-RO:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=ReadonlyMDMFFileURI
 
     def __init__(self, filenode_uri=None):
@@ -653,8 +654,8 @@ def wrap_dirnode_cap(filecap):
 @implementer(IVerifierURI)
 class MDMFDirectoryURIVerifier(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2-MDMF-Verifier:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-MDMF-Verifier:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=MDMFVerifierURI
 
     def __init__(self, filenode_uri=None):
@@ -678,8 +679,8 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI):
 @implementer(IVerifierURI)
 class DirectoryURIVerifier(_DirectoryBaseURI):
 
-    BASE_STRING='URI:DIR2-Verifier:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-Verifier:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=SSKVerifierURI
 
     def __init__(self, filenode_uri=None):
@@ -702,8 +703,8 @@ class DirectoryURIVerifier(_DirectoryBaseURI):
 
 @implementer(IVerifierURI)
 class ImmutableDirectoryURIVerifier(DirectoryURIVerifier):
-    BASE_STRING='URI:DIR2-CHK-Verifier:'
-    BASE_STRING_RE=re.compile('^'+BASE_STRING)
+    BASE_STRING=b'URI:DIR2-CHK-Verifier:'
+    BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
     INNER_URI_CLASS=CHKFileVerifierURI
 
 
@@ -725,12 +726,15 @@ class UnknownURI(object):
         return None
 
 
-ALLEGED_READONLY_PREFIX = 'ro.'
-ALLEGED_IMMUTABLE_PREFIX = 'imm.'
+ALLEGED_READONLY_PREFIX = b'ro.'
+ALLEGED_IMMUTABLE_PREFIX = b'imm.'
 
 def from_string(u, deep_immutable=False, name=u"<unknown name>"):
-    if not isinstance(u, str):
-        raise TypeError("URI must be str: %r" % (u,))
+    """Create URI from either unicode or byte string."""
+    if isinstance(u, unicode):
+        u = u.encode("utf-8")
+    if not isinstance(u, bytes):
+        raise TypeError("URI must be unicode string or bytes: %r" % (u,))
 
     # We allow and check ALLEGED_READONLY_PREFIX or ALLEGED_IMMUTABLE_PREFIX
     # on all URIs, even though we would only strictly need to do so for caps of
@@ -748,62 +752,62 @@ def from_string(u, deep_immutable=False, name=u"<unknown name>"):
 
     error = None
     try:
-        if s.startswith('URI:CHK:'):
+        if s.startswith(b'URI:CHK:'):
             return CHKFileURI.init_from_string(s)
-        elif s.startswith('URI:CHK-Verifier:'):
+        elif s.startswith(b'URI:CHK-Verifier:'):
             return CHKFileVerifierURI.init_from_string(s)
-        elif s.startswith('URI:LIT:'):
+        elif s.startswith(b'URI:LIT:'):
             return LiteralFileURI.init_from_string(s)
-        elif s.startswith('URI:SSK:'):
+        elif s.startswith(b'URI:SSK:'):
             if can_be_writeable:
                 return WriteableSSKFileURI.init_from_string(s)
             kind = "URI:SSK file writecap"
-        elif s.startswith('URI:SSK-RO:'):
+        elif s.startswith(b'URI:SSK-RO:'):
             if can_be_mutable:
                 return ReadonlySSKFileURI.init_from_string(s)
             kind = "URI:SSK-RO readcap to a mutable file"
-        elif s.startswith('URI:SSK-Verifier:'):
+        elif s.startswith(b'URI:SSK-Verifier:'):
             return SSKVerifierURI.init_from_string(s)
-        elif s.startswith('URI:MDMF:'):
+        elif s.startswith(b'URI:MDMF:'):
             if can_be_writeable:
                 return WriteableMDMFFileURI.init_from_string(s)
             kind = "URI:MDMF file writecap"
-        elif s.startswith('URI:MDMF-RO:'):
+        elif s.startswith(b'URI:MDMF-RO:'):
             if can_be_mutable:
                 return ReadonlyMDMFFileURI.init_from_string(s)
             kind = "URI:MDMF-RO readcap to a mutable file"
-        elif s.startswith('URI:MDMF-Verifier:'):
+        elif s.startswith(b'URI:MDMF-Verifier:'):
             return MDMFVerifierURI.init_from_string(s)
-        elif s.startswith('URI:DIR2:'):
+        elif s.startswith(b'URI:DIR2:'):
             if can_be_writeable:
                 return DirectoryURI.init_from_string(s)
             kind = "URI:DIR2 directory writecap"
-        elif s.startswith('URI:DIR2-RO:'):
+        elif s.startswith(b'URI:DIR2-RO:'):
             if can_be_mutable:
                 return ReadonlyDirectoryURI.init_from_string(s)
             kind = "URI:DIR2-RO readcap to a mutable directory"
-        elif s.startswith('URI:DIR2-Verifier:'):
+        elif s.startswith(b'URI:DIR2-Verifier:'):
             return DirectoryURIVerifier.init_from_string(s)
-        elif s.startswith('URI:DIR2-CHK:'):
+        elif s.startswith(b'URI:DIR2-CHK:'):
             return ImmutableDirectoryURI.init_from_string(s)
-        elif s.startswith('URI:DIR2-CHK-Verifier:'):
+        elif s.startswith(b'URI:DIR2-CHK-Verifier:'):
             return ImmutableDirectoryURIVerifier.init_from_string(s)
-        elif s.startswith('URI:DIR2-LIT:'):
+        elif s.startswith(b'URI:DIR2-LIT:'):
             return LiteralDirectoryURI.init_from_string(s)
-        elif s.startswith('URI:DIR2-MDMF:'):
+        elif s.startswith(b'URI:DIR2-MDMF:'):
             if can_be_writeable:
                 return MDMFDirectoryURI.init_from_string(s)
             kind = "URI:DIR2-MDMF directory writecap"
-        elif s.startswith('URI:DIR2-MDMF-RO:'):
+        elif s.startswith(b'URI:DIR2-MDMF-RO:'):
             if can_be_mutable:
                 return ReadonlyMDMFDirectoryURI.init_from_string(s)
             kind = "URI:DIR2-MDMF-RO readcap to a mutable directory"
-        elif s.startswith('URI:DIR2-MDMF-Verifier:'):
+        elif s.startswith(b'URI:DIR2-MDMF-Verifier:'):
             return MDMFDirectoryURIVerifier.init_from_string(s)
-        elif s.startswith('x-tahoe-future-test-writeable:') and not can_be_writeable:
+        elif s.startswith(b'x-tahoe-future-test-writeable:') and not can_be_writeable:
             # For testing how future writeable caps would behave in read-only contexts.
             kind = "x-tahoe-future-test-writeable: testing cap"
-        elif s.startswith('x-tahoe-future-test-mutable:') and not can_be_mutable:
+        elif s.startswith(b'x-tahoe-future-test-mutable:') and not can_be_mutable:
             # For testing how future mutable readcaps would behave in immutable contexts.
             kind = "x-tahoe-future-test-mutable: testing cap"
         else:
@@ -829,18 +833,22 @@ def is_uri(s):
         return False
 
 def is_literal_file_uri(s):
-    if not isinstance(s, str):
+    if isinstance(s, unicode):
+        s = s.encode("utf-8")
+    if not isinstance(s, bytes):
         return False
-    return (s.startswith('URI:LIT:') or
-            s.startswith(ALLEGED_READONLY_PREFIX + 'URI:LIT:') or
-            s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:LIT:'))
+    return (s.startswith(b'URI:LIT:') or
+            s.startswith(ALLEGED_READONLY_PREFIX + b'URI:LIT:') or
+            s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:LIT:'))
 
 def has_uri_prefix(s):
-    if not isinstance(s, str):
+    if isinstance(s, unicode):
+        s = s.encode("utf-8")
+    if not isinstance(s, bytes):
         return False
-    return (s.startswith("URI:") or
-            s.startswith(ALLEGED_READONLY_PREFIX + 'URI:') or
-            s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:'))
+    return (s.startswith(b"URI:") or
+            s.startswith(ALLEGED_READONLY_PREFIX + b'URI:') or
+            s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:'))
 
 
 # These take the same keyword arguments as from_string above.
@@ -850,26 +858,26 @@ def from_string_dirnode(s, **kwargs):
     _assert(IDirnodeURI.providedBy(u))
     return u
 
-registerAdapter(from_string_dirnode, str, IDirnodeURI)
+registerAdapter(from_string_dirnode, bytes, IDirnodeURI)
 
 def from_string_filenode(s, **kwargs):
     u = from_string(s, **kwargs)
     _assert(IFileURI.providedBy(u))
     return u
 
-registerAdapter(from_string_filenode, str, IFileURI)
+registerAdapter(from_string_filenode, bytes, IFileURI)
 
 def from_string_mutable_filenode(s, **kwargs):
     u = from_string(s, **kwargs)
     _assert(IMutableFileURI.providedBy(u))
     return u
-registerAdapter(from_string_mutable_filenode, str, IMutableFileURI)
+registerAdapter(from_string_mutable_filenode, bytes, IMutableFileURI)
 
 def from_string_verifier(s, **kwargs):
     u = from_string(s, **kwargs)
     _assert(IVerifierURI.providedBy(u))
     return u
-registerAdapter(from_string_verifier, str, IVerifierURI)
+registerAdapter(from_string_verifier, bytes, IVerifierURI)
 
 
 def pack_extension(data):
@@ -877,30 +885,32 @@ def pack_extension(data):
     for k in sorted(data.keys()):
         value = data[k]
         if isinstance(value, (int, long)):
-            value = "%d" % value
-        assert isinstance(value, str), k
-        assert re.match(r'^[a-zA-Z_\-]+$', k)
-        pieces.append(k + ':' + hashutil.netstring(value))
-    uri_extension = ''.join(pieces)
+            value = b"%d" % value
+        if isinstance(k, unicode):
+            k = k.encode("utf-8")
+        assert isinstance(value, bytes), k
+        assert re.match(br'^[a-zA-Z_\-]+$', k)
+        pieces.append(k + b':' + hashutil.netstring(value))
+    uri_extension = b''.join(pieces)
     return uri_extension
 
 def unpack_extension(data):
     d = {}
     while data:
-        colon = data.index(':')
+        colon = data.index(b':')
         key = data[:colon]
         data = data[colon+1:]
 
-        colon = data.index(':')
+        colon = data.index(b':')
         number = data[:colon]
         length = int(number)
         data = data[colon+1:]
 
         value = data[:length]
-        assert data[length] == ','
+        assert data[length] == b','
         data = data[length+1:]
 
-        d[key] = value
+        d[key.decode("utf-8")] = value
 
     # convert certain things to numbers
     for intkey in ('size', 'segment_size', 'num_segments',
@@ -914,6 +924,7 @@ def unpack_extension_readable(data):
     unpacked = unpack_extension(data)
     unpacked["UEB_hash"] = hashutil.uri_extension_hash(data)
     for k in sorted(unpacked.keys()):
+        k = k.decode("utf-8")
         if 'hash' in k:
             unpacked[k] = base32.b2a(unpacked[k])
     return unpacked

From 0e15712e347c75ea843629ed6b53afb210d8b2d5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 24 Aug 2020 13:53:27 -0400
Subject: [PATCH 27/39] Tests pass on Python 2 and 3.

---
 src/allmydata/test/test_uri.py | 26 +++++++++++------------
 src/allmydata/uri.py           | 39 +++++++++++++++++-----------------
 2 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py
index 4b9c229f0..005cdf597 100644
--- a/src/allmydata/test/test_uri.py
+++ b/src/allmydata/test/test_uri.py
@@ -202,24 +202,24 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class Extension(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_pack(self):
-        data = {"stuff": b"value",
-                "size": 12,
-                "needed_shares": 3,
-                "big_hash": hashutil.tagged_hash(b"foo", b"bar"),
+        data = {b"stuff": b"value",
+                b"size": 12,
+                b"needed_shares": 3,
+                b"big_hash": hashutil.tagged_hash(b"foo", b"bar"),
                 }
         ext = uri.pack_extension(data)
         d = uri.unpack_extension(ext)
-        self.failUnlessReallyEqual(d["stuff"], b"value")
-        self.failUnlessReallyEqual(d["size"], 12)
-        self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash(b"foo", b"bar"))
+        self.failUnlessReallyEqual(d[b"stuff"], b"value")
+        self.failUnlessReallyEqual(d[b"size"], 12)
+        self.failUnlessReallyEqual(d[b"big_hash"], hashutil.tagged_hash(b"foo", b"bar"))
 
         readable = uri.unpack_extension_readable(ext)
-        self.failUnlessReallyEqual(readable["needed_shares"], 3)
-        self.failUnlessReallyEqual(readable["stuff"], b"value")
-        self.failUnlessReallyEqual(readable["size"], 12)
-        self.failUnlessReallyEqual(readable["big_hash"],
+        self.failUnlessReallyEqual(readable[b"needed_shares"], 3)
+        self.failUnlessReallyEqual(readable[b"stuff"], b"value")
+        self.failUnlessReallyEqual(readable[b"size"], 12)
+        self.failUnlessReallyEqual(readable[b"big_hash"],
                              base32.b2a(hashutil.tagged_hash(b"foo", b"bar")))
-        self.failUnlessReallyEqual(readable["UEB_hash"],
+        self.failUnlessReallyEqual(readable[b"UEB_hash"],
                              base32.b2a(hashutil.uri_extension_hash(ext)))
 
 class Unknown(testutil.ReallyEqualMixin, unittest.TestCase):
@@ -254,7 +254,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase):
 
 class Constraint(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_constraint(self):
-        bad = "http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/"
+        bad = b"http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/"
         self.failUnlessRaises(uri.BadURIError, uri.DirectoryURI.init_from_string, bad)
         fileURI = b'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834'
         uri.CHKFileURI.init_from_string(fileURI)
diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index 70fd80c92..62f4025fb 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -167,7 +167,7 @@ class LiteralFileURI(_BaseURI):
 
     def __init__(self, data=None):
         if data is not None:
-            assert isinstance(data, str)
+            assert isinstance(data, bytes)
             self.data = data
 
     @classmethod
@@ -222,8 +222,8 @@ class WriteableSSKFileURI(_BaseURI):
         return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.writekey, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.writekey, bytes)
+        assert isinstance(self.fingerprint, bytes)
         return b'URI:SSK:%s:%s' % (base32.b2a(self.writekey),
                                    base32.b2a(self.fingerprint))
 
@@ -269,8 +269,8 @@ class ReadonlySSKFileURI(_BaseURI):
         return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.readkey, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.readkey, bytes)
+        assert isinstance(self.fingerprint, bytes)
         return b'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey),
                                       base32.b2a(self.fingerprint))
 
@@ -315,8 +315,8 @@ class SSKVerifierURI(_BaseURI):
         return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.storage_index, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.storage_index, bytes)
+        assert isinstance(self.fingerprint, bytes)
         return b'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index),
                                             base32.b2a(self.fingerprint))
 
@@ -354,8 +354,8 @@ class WriteableMDMFFileURI(_BaseURI):
         return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.writekey, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.writekey, bytes)
+        assert isinstance(self.fingerprint, bytes)
         ret = b'URI:MDMF:%s:%s' % (base32.b2a(self.writekey),
                                    base32.b2a(self.fingerprint))
         return ret
@@ -403,8 +403,8 @@ class ReadonlyMDMFFileURI(_BaseURI):
         return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.readkey, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.readkey, bytes)
+        assert isinstance(self.fingerprint, bytes)
         ret = b'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey),
                                       base32.b2a(self.fingerprint))
         return ret
@@ -450,8 +450,8 @@ class MDMFVerifierURI(_BaseURI):
         return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)))
 
     def to_string(self):
-        assert isinstance(self.storage_index, str)
-        assert isinstance(self.fingerprint, str)
+        assert isinstance(self.storage_index, bytes)
+        assert isinstance(self.fingerprint, bytes)
         ret = b'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index),
                                             base32.b2a(self.fingerprint))
         return ret
@@ -907,14 +907,14 @@ def unpack_extension(data):
         data = data[colon+1:]
 
         value = data[:length]
-        assert data[length] == b','
+        assert data[length:length+1] == b','
         data = data[length+1:]
 
-        d[key.decode("utf-8")] = value
+        d[key] = value
 
     # convert certain things to numbers
-    for intkey in ('size', 'segment_size', 'num_segments',
-                   'needed_shares', 'total_shares'):
+    for intkey in (b'size', b'segment_size', b'num_segments',
+                   b'needed_shares', b'total_shares'):
         if intkey in d:
             d[intkey] = int(d[intkey])
     return d
@@ -922,10 +922,9 @@ def unpack_extension(data):
 
 def unpack_extension_readable(data):
     unpacked = unpack_extension(data)
-    unpacked["UEB_hash"] = hashutil.uri_extension_hash(data)
+    unpacked[b"UEB_hash"] = hashutil.uri_extension_hash(data)
     for k in sorted(unpacked.keys()):
-        k = k.decode("utf-8")
-        if 'hash' in k:
+        if b'hash' in k:
             unpacked[k] = base32.b2a(unpacked[k])
     return unpacked
 

From 2cc21e98937968dadddca6008102805f7c24bdd1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 24 Aug 2020 13:57:46 -0400
Subject: [PATCH 28/39] Finish the port.

---
 newsfragments/3367.minor       |  0
 src/allmydata/test/test_uri.py |  5 +++--
 src/allmydata/uri.py           | 18 ++++++++++++++++++
 src/allmydata/util/_python3.py |  3 ++-
 4 files changed, 23 insertions(+), 3 deletions(-)
 create mode 100644 newsfragments/3367.minor

diff --git a/newsfragments/3367.minor b/newsfragments/3367.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py
index 005cdf597..f89fae151 100644
--- a/src/allmydata/test/test_uri.py
+++ b/src/allmydata/test/test_uri.py
@@ -1,5 +1,6 @@
-
 """
+Tests for allmydata.uri.
+
 Ported to Python 3.
 """
 
@@ -10,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min  # noqa: F401
+    from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min  # noqa: F401
 
 import os
 from twisted.trial import unittest
diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index 62f4025fb..b601226da 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -1,3 +1,21 @@
+"""
+URIs (kinda sorta, really they're capabilities?).
+
+Ported to Python 3.
+
+Methods ending in to_string() are actually to_bytes(), possibly should be fixed
+in follow-up port.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Don't import bytes, to prevent leaks.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min  # noqa: F401
+
 from past.builtins import unicode, long
 
 import re
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 84c08a36c..2351f3707 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
@@ -34,6 +34,7 @@ PORTED_MODULES = [
     "allmydata.immutable.happiness_upload",
     "allmydata.storage.crawler",
     "allmydata.test.common_py3",
+    "allmydata.uri",
     "allmydata.util._python3",
     "allmydata.util.abbreviate",
     "allmydata.util.assertutil",

From 8682550961bb799399adb0571442ff5caf990d09 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:35:25 -0400
Subject: [PATCH 29/39] More passing tests on Python 3.

---
 src/allmydata/test/test_storage_web.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index 9c64c2f45..e8c5dc8a7 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -107,7 +107,6 @@ class MyStorageServer(StorageServer):
 
 class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -130,12 +129,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
         # this sample is before the crawler has started doing anything
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Reserved space: - 0 B (0)", s)
-        self.failUnlessIn("Total buckets: Not computed yet", s)
-        self.failUnlessIn("Next crawl in", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
+        self.failUnlessIn(b"Total buckets: Not computed yet", s)
+        self.failUnlessIn(b"Next crawl in", s)
 
         # give the bucket-counting-crawler one tick to get started. The
         # cpu_slice=0 will force it to yield right after it processes the
@@ -154,8 +153,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn(" Current crawl ", s)
-            self.failUnlessIn(" (next work in ", s)
+            self.failUnlessIn(b" Current crawl ", s)
+            self.failUnlessIn(b" (next work in ", s)
         d.addCallback(_check)
 
         # now give it enough time to complete a full cycle
@@ -166,8 +165,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = orig_cpu_slice
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("Total buckets: 0 (the number of", s)
-            self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
+            self.failUnlessIn(b"Total buckets: 0 (the number of", s)
+            self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
         d.addCallback(_check2)
         return d
 
@@ -228,20 +227,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             # no ETA is available yet
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (next work", s)
+            self.failUnlessIn(b"complete (next work", s)
 
         def _check_2(ignored):
             # one prefix has finished, so an ETA based upon that elapsed time
             # should be available.
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
 
         def _check_3(ignored):
             # two prefixes have finished
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
             d.callback("done")
 
         hooks[0].addCallback(_check_1).addErrback(d.errback)

From 3d18b24967c43433902f8ffbc94b3569f67292b4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:15 -0400
Subject: [PATCH 30/39] Port even more tests to Python 3.

---
 src/allmydata/test/test_storage_web.py | 55 +++++++++++++-------------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index e8c5dc8a7..bface4b13 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -1171,7 +1171,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
 class WebStatus(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -1181,7 +1180,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_no_server(self):
         w = StorageStatus(None)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
+        self.failUnlessIn(b"<h1>No Storage Server Running</h1>", html)
 
     def test_status(self):
         basedir = "storage/WebStatus/status"
@@ -1192,12 +1191,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss, "nickname")
         d = renderDeferred(w)
         def _check_html(html):
-            self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+            self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
             s = remove_tags(html)
-            self.failUnlessIn("Server Nickname: nickname", s)
-            self.failUnlessIn("Server Nodeid: %s"  % base32.b2a(nodeid), s)
-            self.failUnlessIn("Accepting new shares: Yes", s)
-            self.failUnlessIn("Reserved space: - 0 B (0)", s)
+            self.failUnlessIn(b"Server Nickname: nickname", s)
+            self.failUnlessIn(b"Server Nodeid: %s"  % base32.b2a(nodeid), s)
+            self.failUnlessIn(b"Accepting new shares: Yes", s)
+            self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
         d.addCallback(_check_html)
         d.addCallback(lambda ign: renderJSON(w))
         def _check_json(raw):
@@ -1224,11 +1223,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnless(ss.get_available_space() is None)
 
     def test_status_bad_disk_stats(self):
@@ -1244,11 +1243,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnlessEqual(ss.get_available_space(), 0)
 
     def test_status_right_disk_stats(self):
@@ -1281,14 +1280,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss)
         html = renderSynchronously(w)
 
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Total disk space: 5.00 GB", s)
-        self.failUnlessIn("Disk space used: - 1.00 GB", s)
-        self.failUnlessIn("Disk space free (root): 4.00 GB", s)
-        self.failUnlessIn("Disk space free (non-root): 3.00 GB", s)
-        self.failUnlessIn("Reserved space: - 1.00 GB", s)
-        self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s)
+        self.failUnlessIn(b"Total disk space: 5.00 GB", s)
+        self.failUnlessIn(b"Disk space used: - 1.00 GB", s)
+        self.failUnlessIn(b"Disk space free (root): 4.00 GB", s)
+        self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s)
+        self.failUnlessIn(b"Reserved space: - 1.00 GB", s)
+        self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s)
         self.failUnlessEqual(ss.get_available_space(), 2*GB)
 
     def test_readonly(self):
@@ -1298,9 +1297,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
 
     def test_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1309,9 +1308,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_huge_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1320,9 +1319,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_util(self):
         w = StorageStatusElement(None, None)

From 431e939bb82533944aa22890d969d7ca3fca3337 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:52 -0400
Subject: [PATCH 31/39] Finish porting test_storage_web to Python 3.

---
 newsfragments/3395.minor       | 0
 src/allmydata/util/_python3.py | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 newsfragments/3395.minor

diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 6aa1010bc..82ddebc72 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -83,7 +83,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
-    "allmydata.test.test_storage_web",  # partial, WIP
+    "allmydata.test.test_storage_web",
     "allmydata.test.test_time_format",
     "allmydata.test.test_uri",
     "allmydata.test.test_util",

From 637e8a054436d1df486f427a6dd1a5b097f24569 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:59:10 -0400
Subject: [PATCH 32/39] Remove duplication.

---
 src/allmydata/interfaces.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 298c826b7..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,7 +1,5 @@
 from past.builtins import long
 
-from past.builtins import long
-
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
     IPlugin,

From 36177574be07e0ad081fbc2273bd9cef9b780525 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 11:01:04 -0400
Subject: [PATCH 33/39] Fix lint.

---
 src/allmydata/test/test_storage_web.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index bface4b13..19f98851f 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -9,7 +9,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     # Omitted list sinc it broke a test on Python 2. Shouldn't require further
     # work, when we switch to Python 3 we'll be dropping this, anyway.
@@ -19,7 +19,6 @@ import time
 import os.path
 import re
 import json
-from unittest import skipIf
 
 from twisted.trial import unittest
 

From 44126f840c71c21e7e797cc1de37cedb4d1464c7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:36:09 -0400
Subject: [PATCH 34/39] Refactor tox config to reduce duplication between
 py2/py3/coverage

The following tox envs are now defined:

- py27
- py27-coverage
- py36
- py36-coverage
---
 tox.ini | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/tox.ini b/tox.ini
index 98ca90c39..fe3e651b2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,13 +44,32 @@ usedevelop = False
 # We use extras=test to get things like "mock" that are required for our unit
 # tests.
 extras = test
-commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
-         tahoe --version
 
-[testenv:py36]
+setenv =
+       # Define TEST_SUITE in the environment as an aid to constructing the
+       # correct test command below.
+       !py36: TEST_SUITE = allmydata
+       py36: TEST_SUITE = allmydata.test.python3_tests
+
 commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
+	 # As an aid to debugging, dump all of the Python packages and their
+	 # versions that are installed in the test environment.  This is
+	 # particularly useful to get from CI runs - though hopefully the
+	 # version pinning we do limits the variability of this output
+         pip freeze
+
+	 # The tahoe script isn't sufficiently ported for this to succeed on
+	 # Python 3.x yet.
+         !py36: tahoe --version
+
+         !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
+
+	 # measuring coverage is somewhat slower than not measuring coverage
+	 # so only do it on request.
+	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+	 coverage: coverage combine
+         coverage: coverage xml
+
 
 [testenv:integration]
 setenv =
@@ -61,19 +80,6 @@ commands =
          coverage combine
          coverage report
 
-[testenv:coverage]
-# coverage (with --branch) takes about 65% longer to run
-commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
-	 # somewhat.
-         pip freeze
-         tahoe --version
-         coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
-         coverage combine
-         coverage xml
 
 [testenv:codechecks]
 # On macOS, git inside of towncrier needs $HOME.

From 07e33e78a5dcf55aa23cbde6f7edf58d56d120d1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:37:21 -0400
Subject: [PATCH 35/39] Configure CI to use the py36-coverage tox env

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c15eb1746..df181f058 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -285,7 +285,7 @@ jobs:
       # this reporter on Python 3.  So drop that and just specify the
       # reporter.
       TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
 
 
   ubuntu-20.04:

From 7f3192e310fdee14675d9a2b1ce661e1b3b9824f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:46:57 -0400
Subject: [PATCH 36/39] news fragment

---
 newsfragments/3355.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3355.minor

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
new file mode 100644
index 000000000..e69de29bb

From e107e110792033b90f8e13eed1b7caba890d6bee Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:37:37 -0400
Subject: [PATCH 37/39] Remove references to the bare "coverage" tox
 environment

* We stopped using Appveyor a while ago so entirely remove its configuration.
* There's no release step where coverage information is *examined* so why
  collect it?
* Switch GitHub Actions config to py27-coverage tox environment
---
 .appveyor.yml                             | 95 -----------------------
 .github/workflows/ci.yml                  |  4 +-
 docs/how_to_make_a_tahoe-lafs_release.org |  2 +-
 3 files changed, 3 insertions(+), 98 deletions(-)
 delete mode 100644 .appveyor.yml

diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index f6efe785a..000000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-# adapted from https://packaging.python.org/en/latest/appveyor/
-
-environment:
-
-  matrix:
-
-    # For Python versions available on Appveyor, see
-    # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
-    #  DISTUTILS_USE_SDK: "1"
-    #  TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
-
-install:
-  - |
-    %PYTHON%\python.exe -m pip install -U pip
-    %PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
-
-# note:
-# %PYTHON% has: python.exe
-# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
-
-# We have a custom "build" system.  We don't need MSBuild or whatever.
-build: off
-
-# Do not build feature branch with open pull requests.  This is documented but
-# it's not clear it does anything.
-skip_branch_with_pr: true
-
-# This, perhaps, is effective.
-branches:
-  # whitelist
-  only:
-    - 'master'
-
-skip_commits:
- files:
-   # The Windows builds are unaffected by news fragments.
-   - 'newsfragments/*'
-   # Also, all this build junk.
-   - '.circleci/*'
-   - '.lgtm.yml'
-   - '.travis.yml'
-
-# we run from C:\projects\tahoe-lafs
-
-test_script:
-  # Put your test command here.
-  # Note that you must use the environment variable %PYTHON% to refer to
-  # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python version you want to use on PATH.
-  - |
-    %PYTHON%\Scripts\tox.exe -e coverage
-    %PYTHON%\Scripts\tox.exe -e pyinstaller
-  # To verify that the resultant PyInstaller-generated binary executes
-  # cleanly (i.e., that it terminates with an exit code of 0 and isn't
-  # failing due to import/packaging-related errors, etc.).
-  - dist\Tahoe-LAFS\tahoe.exe --version
-
-after_test:
-  # This builds the main tahoe wheel, and wheels for all dependencies.
-  # Again, you only need build.cmd if you're building C extensions for
-  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
-  # interpreter. If _trial_temp still exists, the "pip wheel" fails on
-  # _trial_temp\local_dir (not sure why).
-  - |
-    copy _trial_temp\test.log trial_test_log.txt
-    rd /s /q _trial_temp
-    %PYTHON%\python.exe setup.py bdist_wheel
-    %PYTHON%\python.exe -m pip wheel -w dist .
-  - |
-    %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
-    %PYTHON%\python.exe -m coverage xml -o coverage.xml -i
-    %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-  # "pip wheel -w dist ." puts all the dependency wheels there too
-  # this gives us a zipfile with everything
-  - path: 'dist\*'
-  - path: trial_test_log.txt
-    name: Trial test.log
-  - path: eliot.log
-    name: Eliot test log
-
-on_failure:
-  # Artifacts are not normally uploaded when the job fails.  To get the test
-  # logs, we have to push them ourselves.
-  - ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
-  - ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7cd97dcca..34a4e0875 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,8 +49,8 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e coverage"
-        run: tox -e coverage
+      - name: Run "tox -e py27-coverage"
+        run: tox -e py27-coverage
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1
diff --git a/docs/how_to_make_a_tahoe-lafs_release.org b/docs/how_to_make_a_tahoe-lafs_release.org
index 44b9e3dd1..b3f2a84d7 100644
--- a/docs/how_to_make_a_tahoe-lafs_release.org
+++ b/docs/how_to_make_a_tahoe-lafs_release.org
@@ -36,7 +36,7 @@ people are Release Maintainers:
   - [ ] documentation is ready (see above)
   - [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
   - [ ] build code locally:
-     tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
+     tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
   - [ ] created tarballs (they'll be in dist/ for later comparison)
      tox -e tarballs
   - [ ] release version is reporting itself as intended version

From 6422dba90de0bfd48c41c2a7b68ccd5726a03b36 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:39:49 -0400
Subject: [PATCH 38/39] Say some words about this change

---
 newsfragments/3355.minor | 0
 newsfragments/3355.other | 1 +
 2 files changed, 1 insertion(+)
 delete mode 100644 newsfragments/3355.minor
 create mode 100644 newsfragments/3355.other

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
deleted file mode 100644
index e69de29bb..000000000
diff --git a/newsfragments/3355.other b/newsfragments/3355.other
new file mode 100644
index 000000000..4e854e4dd
--- /dev/null
+++ b/newsfragments/3355.other
@@ -0,0 +1 @@
+The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

From 0ed4f81e62bbb223b6d71beec11b32d8d1cd5476 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:40:27 -0400
Subject: [PATCH 39/39] Replace tabs with spaces

---
 tox.ini | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tox.ini b/tox.ini
index fe3e651b2..2cb59388f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,22 +52,22 @@ setenv =
        py36: TEST_SUITE = allmydata.test.python3_tests
 
 commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
+         # As an aid to debugging, dump all of the Python packages and their
+         # versions that are installed in the test environment.  This is
+         # particularly useful to get from CI runs - though hopefully the
+         # version pinning we do limits the variability of this output
          pip freeze
 
-	 # The tahoe script isn't sufficiently ported for this to succeed on
-	 # Python 3.x yet.
+         # The tahoe script isn't sufficiently ported for this to succeed on
+         # Python 3.x yet.
          !py36: tahoe --version
 
          !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
 
-	 # measuring coverage is somewhat slower than not measuring coverage
-	 # so only do it on request.
-	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
-	 coverage: coverage combine
+         # measuring coverage is somewhat slower than not measuring coverage
+         # so only do it on request.
+         coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+         coverage: coverage combine
          coverage: coverage xml
 
 
@@ -75,8 +75,8 @@ commands =
 setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
-	 # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-	 py.test --coverage -v {posargs:integration}
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         py.test --coverage -v {posargs:integration}
          coverage combine
          coverage report
 
@@ -93,11 +93,11 @@ commands =
          python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
          python misc/coding_tools/check-miscaptures.py
 
-	 # If towncrier.check fails, you forgot to add a towncrier news
-	 # fragment explaining the change in this branch.  Create one at
-	 # `newsfragments/<ticket>.<change type>` with some text for the news
-	 # file.  See pyproject.toml for legal <change type> values.
-	 python -m towncrier.check --pyproject towncrier.pyproject.toml
+         # If towncrier.check fails, you forgot to add a towncrier news
+         # fragment explaining the change in this branch.  Create one at
+         # `newsfragments/<ticket>.<change type>` with some text for the news
+         # file.  See pyproject.toml for legal <change type> values.
+         python -m towncrier.check --pyproject towncrier.pyproject.toml
 
 [testenv:draftnews]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
@@ -116,9 +116,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7
@@ -144,9 +144,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7