From 1c9e4ec842ccb42f670733071e2ac9ed869917fc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:04:52 -0400
Subject: [PATCH 01/21] Move connection_status tests into their own module.

---
 src/allmydata/test/test_connection_status.py | 112 +++++++++++++++++++
 src/allmydata/test/test_connections.py       | 104 -----------------
 2 files changed, 112 insertions(+), 104 deletions(-)
 create mode 100644 src/allmydata/test/test_connection_status.py

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
new file mode 100644
index 000000000..763c14c57
--- /dev/null
+++ b/src/allmydata/test/test_connection_status.py
@@ -0,0 +1,112 @@
+"""
+Tests for allmydata.util.connection_status.
+"""
+
+import mock
+
+from twisted.trial import unittest
+
+from ..util import connection_status
+
+class Status(unittest.TestCase):
+    def test_hint_statuses(self):
+        ncs = connection_status._hint_statuses(["h2","h1"],
+                                               {"h1": "hand1", "h4": "hand4"},
+                                               {"h1": "st1", "h2": "st2",
+                                                "h3": "st3"})
+        self.assertEqual(ncs, {"h1 via hand1": "st1",
+                               "h2": "st2"})
+
+    def test_reconnector_connected(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_others(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_listener(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.listenerStatus = ("listener1", "successful")
+        ci.winningHint = None
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected via listener (listener1)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connecting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "connecting"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary, "Trying to connect")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_waiting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "waiting"
+        ri.lastAttempt = 10
+        ri.nextAttempt = 20
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        with mock.patch("time.time", return_value=12):
+            cs = connection_status.from_foolscap_reconnector(rc, 5)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary,
+                         "Reconnecting in 8 seconds (last attempt 2s ago)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 5)
diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py
index 3e2806dd0..9b5bd7f30 100644
--- a/src/allmydata/test/test_connections.py
+++ b/src/allmydata/test/test_connections.py
@@ -7,7 +7,6 @@ from foolscap.connections import tcp
 from ..node import PrivacyError, config_from_string
 from ..node import create_connection_handlers
 from ..node import create_main_tub, _tub_portlocation
-from ..util import connection_status
 from ..util.i2p_provider import create as create_i2p_provider
 from ..util.tor_provider import create as create_tor_provider
 
@@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
             str(ctx.exception),
             "tub.location includes tcp: hint",
         )
-
-class Status(unittest.TestCase):
-    def test_hint_statuses(self):
-        ncs = connection_status._hint_statuses(["h2","h1"],
-                                               {"h1": "hand1", "h4": "hand4"},
-                                               {"h1": "st1", "h2": "st2",
-                                                "h3": "st3"})
-        self.assertEqual(ncs, {"h1 via hand1": "st1",
-                               "h2": "st2"})
-
-    def test_reconnector_connected(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_others(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_listener(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.listenerStatus = ("listener1", "successful")
-        ci.winningHint = None
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected via listener (listener1)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connecting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "connecting"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary, "Trying to connect")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_waiting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "waiting"
-        ri.lastAttempt = 10
-        ri.nextAttempt = 20
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        with mock.patch("time.time", return_value=12):
-            cs = connection_status.from_foolscap_reconnector(rc, 5)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary,
-                         "Reconnecting in 8 seconds (last attempt 2s ago)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 5)

From 243d02ecb1a01216fc16331ec25a04186dfce063 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:08:51 -0400
Subject: [PATCH 02/21] Port to Python 3.

---
 src/allmydata/test/test_connection_status.py | 10 ++++++++++
 src/allmydata/util/_python3.py               |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 763c14c57..2bd8bf6ab 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -1,6 +1,16 @@
 """
 Tests for allmydata.util.connection_status.
+
+Port to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import mock
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 091c248af..f074927b6 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -50,6 +50,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",

From a08cde9a4da3bf636560fc72e2297a371953b38e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:30:16 -0400
Subject: [PATCH 03/21] Port to Python 3.

---
 src/allmydata/interfaces.py             |  1 +
 src/allmydata/util/_python3.py          |  1 +
 src/allmydata/util/connection_status.py | 22 ++++++++++++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 36eb55bfb..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,3 +1,4 @@
+from past.builtins import long
 
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f074927b6..494d5e161 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
     "allmydata.util.gcutil",
diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index 44c12f220..0e8595e81 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -1,3 +1,18 @@
+"""
+Parse connection status from Foolscap.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus
@@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
 
 def from_foolscap_reconnector(rc, last_received):
     ri = rc.getReconnectionInfo()
-    # See foolscap/reconnector.py, ReconnectionInfo, for details about
-    # possible states.
+    # See foolscap/reconnector.py, ReconnectionInfo, for details about possible
+    # states. The returned result is a native string, it seems, so convert to
+    # unicode.
     state = ri.state
+    if isinstance(state, bytes):  # Python 2
+        state = str(state, "ascii")
     if state == "unstarted":
         return ConnectionStatus.unstarted()
 

From e24c21bef779de6d1d419b2135231f628a0bcc20 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:38:58 -0400
Subject: [PATCH 04/21] Make configutil tests more standalone, and less
 repetitive.

---
 src/allmydata/test/cli/test_create.py | 27 ++++++++
 src/allmydata/test/test_configutil.py | 90 +++++++--------------------
 2 files changed, 49 insertions(+), 68 deletions(-)

diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py
index 75162c39e..f013c0205 100644
--- a/src/allmydata/test/cli/test_create.py
+++ b/src/allmydata/test/cli/test_create.py
@@ -6,6 +6,8 @@ from twisted.python import usage
 from allmydata.util import configutil
 from ..common_util import run_cli, parse_cli
 from ...scripts import create_node
+from ... import client
+
 
 def read_config(basedir):
     tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@@ -33,6 +35,31 @@ class Config(unittest.TestCase):
             e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
             self.assertIn("option %s not recognized" % (option,), str(e))
 
+    def test_create_client_config(self):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+
+        with open(fname, 'w') as f:
+            opts = {"nickname": "nick",
+                    "webport": "tcp:3456",
+                    "hide-ip": False,
+                    "listen": "none",
+                    "shares-needed": "1",
+                    "shares-happy": "1",
+                    "shares-total": "1",
+                    }
+            create_node.write_node_config(f, opts)
+            create_node.write_client_config(f, opts)
+
+        config = configutil.get_config(fname)
+        # should succeed, no exceptions
+        configutil.validate_config(
+            fname,
+            config,
+            client._valid_config(),
+        )
+
     @defer.inlineCallbacks
     def test_client(self):
         basedir = self.mktemp()
diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index 45eb6ac25..ffb5f5320 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -3,12 +3,9 @@ import os.path
 from twisted.trial import unittest
 
 from allmydata.util import configutil
-from allmydata.test.no_network import GridTestMixin
-from ..scripts import create_node
-from .. import client
 
 
-class ConfigUtilTests(GridTestMixin, unittest.TestCase):
+class ConfigUtilTests(unittest.TestCase):
     def setUp(self):
         super(ConfigUtilTests, self).setUp()
         self.static_valid_config = configutil.ValidConfiguration(
@@ -20,10 +17,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
         )
 
+    def create_tahoe_cfg(self, cfg):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+        with open(fname, "w") as f:
+            f.write(cfg)
+        return fname
+
     def test_config_utils(self):
-        self.basedir = "cli/ConfigUtilTests/test-config-utils"
-        self.set_up_grid(oneshare=True)
-        tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
+        tahoe_cfg = self.create_tahoe_cfg("""\
+[node]
+nickname = client-0
+web.port = adopt-socket:fd=5
+[storage]
+enabled = false
+""")
 
         # test that at least one option was read correctly
         config = configutil.get_config(tahoe_cfg)
@@ -45,12 +54,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
 
     def test_config_validation_success(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -66,12 +70,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         validation but are matched by the dynamic validation is considered
         valid.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -82,12 +81,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         )
 
     def test_config_validation_invalid_item(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -103,12 +97,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -124,12 +113,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -145,12 +129,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section, item pair that is matched by neither the
         static nor dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -160,28 +139,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             self.dynamic_valid_config,
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
-
-    def test_create_client_config(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            opts = {"nickname": "nick",
-                    "webport": "tcp:3456",
-                    "hide-ip": False,
-                    "listen": "none",
-                    "shares-needed": "1",
-                    "shares-happy": "1",
-                    "shares-total": "1",
-                    }
-            create_node.write_node_config(f, opts)
-            create_node.write_client_config(f, opts)
-
-        config = configutil.get_config(fname)
-        # should succeed, no exceptions
-        configutil.validate_config(
-            fname,
-            config,
-            client._valid_config(),
-        )

From babe2dbc85d8c6d87bec5e1a919faf71f854b596 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:45:52 -0400
Subject: [PATCH 05/21] Port to Python 3.

---
 src/allmydata/test/test_configutil.py | 15 +++++++++++++++
 src/allmydata/util/_python3.py        |  1 +
 2 files changed, 16 insertions(+)

diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index ffb5f5320..c57381289 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -1,3 +1,18 @@
+"""
+Tests for allmydata.util.configutil.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Omitted dict, cause worried about interactions.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 
 from twisted.trial import unittest
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 494d5e161..b6c4b6c87 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -51,6 +51,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_configutil",
     "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",

From 11b934120c1b5f279e4c84c1d94de8bec9ea2bc8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:49:59 -0400
Subject: [PATCH 06/21] Port to Python 3.

---
 src/allmydata/util/_python3.py   |  1 +
 src/allmydata/util/configutil.py | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index b6c4b6c87..947fcf00b 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.configutil",
     "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 3699db35d..295605b65 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,5 +1,13 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
-from ConfigParser import SafeConfigParser
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from configparser import SafeConfigParser
 
 import attr
 
@@ -13,11 +21,7 @@ class UnknownConfigError(Exception):
 
 def get_config(tahoe_cfg):
     config = SafeConfigParser()
-    with open(tahoe_cfg, "rb") as f:
-        # Skip any initial Byte Order Mark. Since this is an ordinary file, we
-        # don't need to handle incomplete reads, and can assume seekability.
-        if f.read(3) != '\xEF\xBB\xBF':
-            f.seek(0)
+    with open(tahoe_cfg, "r") as f:
         config.readfp(f)
     return config
 
@@ -28,7 +32,7 @@ def set_config(config, section, option, value):
     assert config.get(section, option) == value
 
 def write_config(tahoe_cfg, config):
-    with open(tahoe_cfg, "wb") as f:
+    with open(tahoe_cfg, "w") as f:
         config.write(f)
 
 def validate_config(fname, cfg, valid_config):

From 277298050f16ab3794868f377b6a617820baefe7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:51:27 -0400
Subject: [PATCH 07/21] News file and ratchet.

---
 misc/python3/ratchet-passing | 13 +++++++++++++
 newsfragments/3377.minor     |  0
 2 files changed, 13 insertions(+)
 create mode 100644 newsfragments/3377.minor

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bbc6e6c7e..e6b0d60c2 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -24,6 +24,19 @@ allmydata.test.test_base62.Base62.test_known_values
 allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
 allmydata.test.test_base62.Base62.test_odd_sizes
 allmydata.test.test_base62.Base62.test_roundtrip
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success
+allmydata.test.test_configutil.ConfigUtilTests.test_config_utils
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success
+allmydata.test.test_connection_status.Status.test_hint_statuses
+allmydata.test.test_connection_status.Status.test_reconnector_connected
+allmydata.test.test_connection_status.Status.test_reconnector_connected_listener
+allmydata.test.test_connection_status.Status.test_reconnector_connected_others
+allmydata.test.test_connection_status.Status.test_reconnector_connecting
+allmydata.test.test_connection_status.Status.test_reconnector_waiting
 allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_key_serialization
diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor
new file mode 100644
index 000000000..e69de29bb

From 10378541d72a6853b4cc1f3c86482979f14b3827 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 16:40:00 -0400
Subject: [PATCH 08/21] Use Python 2 ConfigParser, so correct exceptions get
 raised.

---
 src/allmydata/util/configutil.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 295605b65..79980b006 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -7,10 +7,17 @@ from future.utils import PY2
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from configparser import SafeConfigParser
+if PY2:
+    # In theory on Python 2 configparser also works, but then code gets the
+    # wrong exceptions and they don't get handled. So just use native parser
+    # for now.
+    from ConfigParser import SafeConfigParser
+else:
+    from configparser import SafeConfigParser
 
 import attr
 
+
 class UnknownConfigError(Exception):
     """
     An unknown config item was found.

From af61571fa61616e4bdd43326919075d46eeaf6cc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 15:56:08 -0400
Subject: [PATCH 09/21] Accidentally passing test :(

---
 misc/python3/ratchet-passing | 1 +
 1 file changed, 1 insertion(+)

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bf56e0de2..01e58a2fa 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -206,6 +206,7 @@ allmydata.test.test_statistics.Statistics.test_pr_file_loss
 allmydata.test.test_statistics.Statistics.test_repair_cost
 allmydata.test.test_statistics.Statistics.test_repair_count_pmf
 allmydata.test.test_statistics.Statistics.test_survival_pmf
+allmydata.test.test_stats.CPUUsage.test_monitor
 allmydata.test.test_time_format.TimeFormat.test_epoch
 allmydata.test.test_time_format.TimeFormat.test_epoch_in_London
 allmydata.test.test_time_format.TimeFormat.test_format_delta

From b0c4f6d2abaea2d2a2ca5408072cb8ba68fb827a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 16:30:27 -0400
Subject: [PATCH 10/21] Fix Python 2 tests.

---
 src/allmydata/util/configutil.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 79980b006..1a1a93f18 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,3 +1,10 @@
+"""
+Read/write config files.
+
+Configuration is returned as native strings.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -5,7 +12,9 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # We don't do open(), because we want files to read/write native strs when
+    # we do "r" or "w".
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 if PY2:
     # In theory on Python 2 configparser also works, but then code gets the
@@ -27,8 +36,17 @@ class UnknownConfigError(Exception):
 
 
 def get_config(tahoe_cfg):
+    """Load the config, returning a SafeConfigParser.
+
+    Configuration is returned as native strings.
+    """
     config = SafeConfigParser()
     with open(tahoe_cfg, "r") as f:
+        # On Python 2, where we read in bytes, skip any initial Byte Order
+        # Mark. Since this is an ordinary file, we don't need to handle
+        # incomplete reads, and can assume seekability.
+        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
+            f.seek(0)
         config.readfp(f)
     return config
 

From 8682550961bb799399adb0571442ff5caf990d09 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:35:25 -0400
Subject: [PATCH 11/21] More passing tests on Python 3.

---
 src/allmydata/test/test_storage_web.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index 9c64c2f45..e8c5dc8a7 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -107,7 +107,6 @@ class MyStorageServer(StorageServer):
 
 class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -130,12 +129,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
         # this sample is before the crawler has started doing anything
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Reserved space: - 0 B (0)", s)
-        self.failUnlessIn("Total buckets: Not computed yet", s)
-        self.failUnlessIn("Next crawl in", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
+        self.failUnlessIn(b"Total buckets: Not computed yet", s)
+        self.failUnlessIn(b"Next crawl in", s)
 
         # give the bucket-counting-crawler one tick to get started. The
         # cpu_slice=0 will force it to yield right after it processes the
@@ -154,8 +153,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn(" Current crawl ", s)
-            self.failUnlessIn(" (next work in ", s)
+            self.failUnlessIn(b" Current crawl ", s)
+            self.failUnlessIn(b" (next work in ", s)
         d.addCallback(_check)
 
         # now give it enough time to complete a full cycle
@@ -166,8 +165,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = orig_cpu_slice
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("Total buckets: 0 (the number of", s)
-            self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
+            self.failUnlessIn(b"Total buckets: 0 (the number of", s)
+            self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
         d.addCallback(_check2)
         return d
 
@@ -228,20 +227,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             # no ETA is available yet
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (next work", s)
+            self.failUnlessIn(b"complete (next work", s)
 
         def _check_2(ignored):
             # one prefix has finished, so an ETA based upon that elapsed time
             # should be available.
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
 
         def _check_3(ignored):
             # two prefixes have finished
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
             d.callback("done")
 
         hooks[0].addCallback(_check_1).addErrback(d.errback)

From 3d18b24967c43433902f8ffbc94b3569f67292b4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:15 -0400
Subject: [PATCH 12/21] Port even more tests to Python 3.

---
 src/allmydata/test/test_storage_web.py | 55 +++++++++++++-------------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index e8c5dc8a7..bface4b13 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -1171,7 +1171,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
 class WebStatus(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -1181,7 +1180,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_no_server(self):
         w = StorageStatus(None)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
+        self.failUnlessIn(b"<h1>No Storage Server Running</h1>", html)
 
     def test_status(self):
         basedir = "storage/WebStatus/status"
@@ -1192,12 +1191,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss, "nickname")
         d = renderDeferred(w)
         def _check_html(html):
-            self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+            self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
             s = remove_tags(html)
-            self.failUnlessIn("Server Nickname: nickname", s)
-            self.failUnlessIn("Server Nodeid: %s"  % base32.b2a(nodeid), s)
-            self.failUnlessIn("Accepting new shares: Yes", s)
-            self.failUnlessIn("Reserved space: - 0 B (0)", s)
+            self.failUnlessIn(b"Server Nickname: nickname", s)
+            self.failUnlessIn(b"Server Nodeid: %s"  % base32.b2a(nodeid), s)
+            self.failUnlessIn(b"Accepting new shares: Yes", s)
+            self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
         d.addCallback(_check_html)
         d.addCallback(lambda ign: renderJSON(w))
         def _check_json(raw):
@@ -1224,11 +1223,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnless(ss.get_available_space() is None)
 
     def test_status_bad_disk_stats(self):
@@ -1244,11 +1243,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnlessEqual(ss.get_available_space(), 0)
 
     def test_status_right_disk_stats(self):
@@ -1281,14 +1280,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss)
         html = renderSynchronously(w)
 
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Total disk space: 5.00 GB", s)
-        self.failUnlessIn("Disk space used: - 1.00 GB", s)
-        self.failUnlessIn("Disk space free (root): 4.00 GB", s)
-        self.failUnlessIn("Disk space free (non-root): 3.00 GB", s)
-        self.failUnlessIn("Reserved space: - 1.00 GB", s)
-        self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s)
+        self.failUnlessIn(b"Total disk space: 5.00 GB", s)
+        self.failUnlessIn(b"Disk space used: - 1.00 GB", s)
+        self.failUnlessIn(b"Disk space free (root): 4.00 GB", s)
+        self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s)
+        self.failUnlessIn(b"Reserved space: - 1.00 GB", s)
+        self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s)
         self.failUnlessEqual(ss.get_available_space(), 2*GB)
 
     def test_readonly(self):
@@ -1298,9 +1297,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
 
     def test_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1309,9 +1308,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_huge_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1320,9 +1319,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_util(self):
         w = StorageStatusElement(None, None)

From 431e939bb82533944aa22890d969d7ca3fca3337 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:52 -0400
Subject: [PATCH 13/21] Finish porting test_storage_web to Python 3.

---
 newsfragments/3395.minor       | 0
 src/allmydata/util/_python3.py | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 newsfragments/3395.minor

diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 6aa1010bc..82ddebc72 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -83,7 +83,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
-    "allmydata.test.test_storage_web",  # partial, WIP
+    "allmydata.test.test_storage_web",
     "allmydata.test.test_time_format",
     "allmydata.test.test_uri",
     "allmydata.test.test_util",

From 637e8a054436d1df486f427a6dd1a5b097f24569 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:59:10 -0400
Subject: [PATCH 14/21] Remove duplication.

---
 src/allmydata/interfaces.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 298c826b7..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,7 +1,5 @@
 from past.builtins import long
 
-from past.builtins import long
-
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
     IPlugin,

From 36177574be07e0ad081fbc2273bd9cef9b780525 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 11:01:04 -0400
Subject: [PATCH 15/21] Fix lint.

---
 src/allmydata/test/test_storage_web.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index bface4b13..19f98851f 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -9,7 +9,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     # Omitted list sinc it broke a test on Python 2. Shouldn't require further
     # work, when we switch to Python 3 we'll be dropping this, anyway.
@@ -19,7 +19,6 @@ import time
 import os.path
 import re
 import json
-from unittest import skipIf
 
 from twisted.trial import unittest
 

From 44126f840c71c21e7e797cc1de37cedb4d1464c7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:36:09 -0400
Subject: [PATCH 16/21] Refactor tox config to reduce duplication between
 py2/py3/coverage

The following tox envs are now defined:

- py27
- py27-coverage
- py36
- py36-coverage
---
 tox.ini | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/tox.ini b/tox.ini
index 98ca90c39..fe3e651b2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,13 +44,32 @@ usedevelop = False
 # We use extras=test to get things like "mock" that are required for our unit
 # tests.
 extras = test
-commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
-         tahoe --version
 
-[testenv:py36]
+setenv =
+       # Define TEST_SUITE in the environment as an aid to constructing the
+       # correct test command below.
+       !py36: TEST_SUITE = allmydata
+       py36: TEST_SUITE = allmydata.test.python3_tests
+
 commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
+	 # As an aid to debugging, dump all of the Python packages and their
+	 # versions that are installed in the test environment.  This is
+	 # particularly useful to get from CI runs - though hopefully the
+	 # version pinning we do limits the variability of this output
+         pip freeze
+
+	 # The tahoe script isn't sufficiently ported for this to succeed on
+	 # Python 3.x yet.
+         !py36: tahoe --version
+
+         !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
+
+	 # measuring coverage is somewhat slower than not measuring coverage
+	 # so only do it on request.
+	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+	 coverage: coverage combine
+         coverage: coverage xml
+
 
 [testenv:integration]
 setenv =
@@ -61,19 +80,6 @@ commands =
          coverage combine
          coverage report
 
-[testenv:coverage]
-# coverage (with --branch) takes about 65% longer to run
-commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
-	 # somewhat.
-         pip freeze
-         tahoe --version
-         coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
-         coverage combine
-         coverage xml
 
 [testenv:codechecks]
 # On macOS, git inside of towncrier needs $HOME.

From 07e33e78a5dcf55aa23cbde6f7edf58d56d120d1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:37:21 -0400
Subject: [PATCH 17/21] Configure CI to use the py36-coverage tox env

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c15eb1746..df181f058 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -285,7 +285,7 @@ jobs:
       # this reporter on Python 3.  So drop that and just specify the
       # reporter.
       TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
 
 
   ubuntu-20.04:

From 7f3192e310fdee14675d9a2b1ce661e1b3b9824f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:46:57 -0400
Subject: [PATCH 18/21] news fragment

---
 newsfragments/3355.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3355.minor

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
new file mode 100644
index 000000000..e69de29bb

From e107e110792033b90f8e13eed1b7caba890d6bee Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:37:37 -0400
Subject: [PATCH 19/21] Remove references to the bare "coverage" tox
 environment

* We stopped using Appveyor a while ago so entirely remove its configuration.
* There's no release step where coverage information is *examined* so why
  collect it?
* Switch GitHub Actions config to py27-coverage tox environment
---
 .appveyor.yml                             | 95 -----------------------
 .github/workflows/ci.yml                  |  4 +-
 docs/how_to_make_a_tahoe-lafs_release.org |  2 +-
 3 files changed, 3 insertions(+), 98 deletions(-)
 delete mode 100644 .appveyor.yml

diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index f6efe785a..000000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-# adapted from https://packaging.python.org/en/latest/appveyor/
-
-environment:
-
-  matrix:
-
-    # For Python versions available on Appveyor, see
-    # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
-    #  DISTUTILS_USE_SDK: "1"
-    #  TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
-
-install:
-  - |
-    %PYTHON%\python.exe -m pip install -U pip
-    %PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
-
-# note:
-# %PYTHON% has: python.exe
-# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
-
-# We have a custom "build" system.  We don't need MSBuild or whatever.
-build: off
-
-# Do not build feature branch with open pull requests.  This is documented but
-# it's not clear it does anything.
-skip_branch_with_pr: true
-
-# This, perhaps, is effective.
-branches:
-  # whitelist
-  only:
-    - 'master'
-
-skip_commits:
- files:
-   # The Windows builds are unaffected by news fragments.
-   - 'newsfragments/*'
-   # Also, all this build junk.
-   - '.circleci/*'
-   - '.lgtm.yml'
-   - '.travis.yml'
-
-# we run from C:\projects\tahoe-lafs
-
-test_script:
-  # Put your test command here.
-  # Note that you must use the environment variable %PYTHON% to refer to
-  # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python version you want to use on PATH.
-  - |
-    %PYTHON%\Scripts\tox.exe -e coverage
-    %PYTHON%\Scripts\tox.exe -e pyinstaller
-  # To verify that the resultant PyInstaller-generated binary executes
-  # cleanly (i.e., that it terminates with an exit code of 0 and isn't
-  # failing due to import/packaging-related errors, etc.).
-  - dist\Tahoe-LAFS\tahoe.exe --version
-
-after_test:
-  # This builds the main tahoe wheel, and wheels for all dependencies.
-  # Again, you only need build.cmd if you're building C extensions for
-  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
-  # interpreter. If _trial_temp still exists, the "pip wheel" fails on
-  # _trial_temp\local_dir (not sure why).
-  - |
-    copy _trial_temp\test.log trial_test_log.txt
-    rd /s /q _trial_temp
-    %PYTHON%\python.exe setup.py bdist_wheel
-    %PYTHON%\python.exe -m pip wheel -w dist .
-  - |
-    %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
-    %PYTHON%\python.exe -m coverage xml -o coverage.xml -i
-    %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-  # "pip wheel -w dist ." puts all the dependency wheels there too
-  # this gives us a zipfile with everything
-  - path: 'dist\*'
-  - path: trial_test_log.txt
-    name: Trial test.log
-  - path: eliot.log
-    name: Eliot test log
-
-on_failure:
-  # Artifacts are not normally uploaded when the job fails.  To get the test
-  # logs, we have to push them ourselves.
-  - ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
-  - ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7cd97dcca..34a4e0875 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,8 +49,8 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e coverage"
-        run: tox -e coverage
+      - name: Run "tox -e py27-coverage"
+        run: tox -e py27-coverage
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1
diff --git a/docs/how_to_make_a_tahoe-lafs_release.org b/docs/how_to_make_a_tahoe-lafs_release.org
index 44b9e3dd1..b3f2a84d7 100644
--- a/docs/how_to_make_a_tahoe-lafs_release.org
+++ b/docs/how_to_make_a_tahoe-lafs_release.org
@@ -36,7 +36,7 @@ people are Release Maintainers:
   - [ ] documentation is ready (see above)
   - [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
   - [ ] build code locally:
-     tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
+     tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
   - [ ] created tarballs (they'll be in dist/ for later comparison)
      tox -e tarballs
   - [ ] release version is reporting itself as intended version

From 6422dba90de0bfd48c41c2a7b68ccd5726a03b36 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:39:49 -0400
Subject: [PATCH 20/21] Say some words about this change

---
 newsfragments/3355.minor | 0
 newsfragments/3355.other | 1 +
 2 files changed, 1 insertion(+)
 delete mode 100644 newsfragments/3355.minor
 create mode 100644 newsfragments/3355.other

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
deleted file mode 100644
index e69de29bb..000000000
diff --git a/newsfragments/3355.other b/newsfragments/3355.other
new file mode 100644
index 000000000..4e854e4dd
--- /dev/null
+++ b/newsfragments/3355.other
@@ -0,0 +1 @@
+The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

From 0ed4f81e62bbb223b6d71beec11b32d8d1cd5476 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:40:27 -0400
Subject: [PATCH 21/21] Replace tabs with spaces

---
 tox.ini | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tox.ini b/tox.ini
index fe3e651b2..2cb59388f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,22 +52,22 @@ setenv =
        py36: TEST_SUITE = allmydata.test.python3_tests
 
 commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
+         # As an aid to debugging, dump all of the Python packages and their
+         # versions that are installed in the test environment.  This is
+         # particularly useful to get from CI runs - though hopefully the
+         # version pinning we do limits the variability of this output
          pip freeze
 
-	 # The tahoe script isn't sufficiently ported for this to succeed on
-	 # Python 3.x yet.
+         # The tahoe script isn't sufficiently ported for this to succeed on
+         # Python 3.x yet.
          !py36: tahoe --version
 
          !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
 
-	 # measuring coverage is somewhat slower than not measuring coverage
-	 # so only do it on request.
-	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
-	 coverage: coverage combine
+         # measuring coverage is somewhat slower than not measuring coverage
+         # so only do it on request.
+         coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+         coverage: coverage combine
          coverage: coverage xml
 
 
@@ -75,8 +75,8 @@ commands =
 setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
-	 # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-	 py.test --coverage -v {posargs:integration}
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         py.test --coverage -v {posargs:integration}
          coverage combine
          coverage report
 
@@ -93,11 +93,11 @@ commands =
          python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
          python misc/coding_tools/check-miscaptures.py
 
-	 # If towncrier.check fails, you forgot to add a towncrier news
-	 # fragment explaining the change in this branch.  Create one at
-	 # `newsfragments/<ticket>.<change type>` with some text for the news
-	 # file.  See pyproject.toml for legal <change type> values.
-	 python -m towncrier.check --pyproject towncrier.pyproject.toml
+         # If towncrier.check fails, you forgot to add a towncrier news
+         # fragment explaining the change in this branch.  Create one at
+         # `newsfragments/<ticket>.<change type>` with some text for the news
+         # file.  See pyproject.toml for legal <change type> values.
+         python -m towncrier.check --pyproject towncrier.pyproject.toml
 
 [testenv:draftnews]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
@@ -116,9 +116,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7
@@ -144,9 +144,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7