From 7516a5526e6561ca3c9f3306b7d46ed06f70047b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 12:57:46 -0400
Subject: [PATCH 01/59] Port to Python 3.

---
 src/allmydata/interfaces.py    | 16 ++++++++++++++++
 src/allmydata/util/_python3.py |  1 +
 2 files changed, 17 insertions(+)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 36eb55bfb..7b02ceea9 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,3 +1,19 @@
+"""
+Interfaces for Tahoe-LAFS.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Don't import object/str/dict/etc. types, so we don't break any interfaces.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, max, min  # noqa: F401
+
+from past.builtins import long
 
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 091c248af..c3de47608 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -22,6 +22,7 @@ PORTED_MODULES = [
     "allmydata.crypto.rsa",
     "allmydata.crypto.util",
     "allmydata.hashtree",
+    "allmydata.interfaces",
     "allmydata.test.common_py3",
     "allmydata.util._python3",
     "allmydata.util.abbreviate",

From 5384768f7608dd074a5c51c798e87b9f1cda6db2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:08:53 -0400
Subject: [PATCH 02/59] Port to Python 3.

---
 src/allmydata/codec.py           | 21 +++++++++++++++++----
 src/allmydata/test/test_codec.py | 25 +++++++++++++++++++------
 src/allmydata/util/_python3.py   |  2 ++
 3 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py
index 1ceb146e4..048355871 100644
--- a/src/allmydata/codec.py
+++ b/src/allmydata/codec.py
@@ -1,4 +1,17 @@
-# -*- test-case-name: allmydata.test.test_encode_share -*-
+"""
+CRS encoding and decoding.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from future.utils import native_str
 
 from zope.interface import implementer
 from twisted.internet import defer
@@ -27,8 +40,8 @@ class CRSEncoder(object):
         return (self.data_size, self.required_shares, self.max_shares)
 
     def get_serialized_params(self):
-        return "%d-%d-%d" % (self.data_size, self.required_shares,
-                             self.max_shares)
+        return native_str("%d-%d-%d" % (self.data_size, self.required_shares,
+                                        self.max_shares))
 
     def get_block_size(self):
         return self.share_size
@@ -37,7 +50,7 @@ class CRSEncoder(object):
         precondition(desired_share_ids is None or len(desired_share_ids) <= self.max_shares, desired_share_ids, self.max_shares)
 
         if desired_share_ids is None:
-            desired_share_ids = range(self.max_shares)
+            desired_share_ids = list(range(self.max_shares))
 
         for inshare in inshares:
             assert len(inshare) == self.share_size, (len(inshare), self.share_size, self.data_size, self.required_shares)
diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py
index 706e62dff..3076a322e 100644
--- a/src/allmydata/test/test_codec.py
+++ b/src/allmydata/test/test_codec.py
@@ -1,3 +1,16 @@
+"""
+Tests for allmydata.codec.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 from twisted.trial import unittest
@@ -23,7 +36,7 @@ class T(unittest.TestCase):
         d.addCallback(_done_encoding_all)
         if fewer_shares is not None:
             # also validate that the desired_shareids= parameter works
-            desired_shareids = random.sample(range(max_shares), fewer_shares)
+            desired_shareids = random.sample(list(range(max_shares)), fewer_shares)
             d.addCallback(lambda res: enc.encode(data0s, desired_shareids))
             def _check_fewer_shares(some_shares_and_their_shareids):
                 (some_shares, their_shareids) = some_shares_and_their_shareids
@@ -38,11 +51,11 @@ class T(unittest.TestCase):
             return d1
 
         def _check_data(decoded_shares):
-            self.failUnlessEqual(len(''.join(decoded_shares)), len(''.join(data0s)))
+            self.failUnlessEqual(len(b''.join(decoded_shares)), len(b''.join(data0s)))
             self.failUnlessEqual(len(decoded_shares), len(data0s))
             for (i, (x, y)) in enumerate(zip(data0s, decoded_shares)):
                 self.failUnlessEqual(x, y, "%s: %r != %r....  first share was %r" % (str(i), x, y, data0s[0],))
-            self.failUnless(''.join(decoded_shares) == ''.join(data0s), "%s" % ("???",))
+            self.failUnless(b''.join(decoded_shares) == b''.join(data0s), "%s" % ("???",))
             # 0data0sclipped = tuple(data0s)
             # data0sclipped[-1] =
             # self.failUnless(tuple(decoded_shares) == tuple(data0s))
@@ -59,7 +72,7 @@ class T(unittest.TestCase):
         def _decode_some_random(res):
             log.msg("_decode_some_random")
             # use a randomly-selected minimal subset
-            l = random.sample(zip(self.shares, self.shareids), required_shares)
+            l = random.sample(list(zip(self.shares, self.shareids)), required_shares)
             some_shares = [ x[0] for x in l ]
             some_shareids = [ x[1] for x in l ]
             return _decode((some_shares, some_shareids))
@@ -70,10 +83,10 @@ class T(unittest.TestCase):
             log.msg("_decode_multiple")
             # make sure we can re-use the decoder object
             shares1 = random.sample(self.shares, required_shares)
-            sharesl1 = random.sample(zip(self.shares, self.shareids), required_shares)
+            sharesl1 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
             shares1 = [ x[0] for x in sharesl1 ]
             shareids1 = [ x[1] for x in sharesl1 ]
-            sharesl2 = random.sample(zip(self.shares, self.shareids), required_shares)
+            sharesl2 = random.sample(list(zip(self.shares, self.shareids)), required_shares)
             shares2 = [ x[0] for x in sharesl2 ]
             shareids2 = [ x[1] for x in sharesl2 ]
             dec = CRSDecoder()
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index c3de47608..2b18eaa1d 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -15,6 +15,7 @@ if PY2:
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
+    "allmydata.codec",
     "allmydata.crypto",
     "allmydata.crypto.aes",
     "allmydata.crypto.ed25519",
@@ -51,6 +52,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_codec",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",

From 43e36ebcb68a75161e807829f95de67310179c28 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:09:25 -0400
Subject: [PATCH 03/59] Ignore Futurize-generated backup files.

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 8191c173b..1af859854 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ venv
 *~
 *.DS_Store
 .*.kate-swp
+*.bak
 
 /build/
 /support/

From 9c0b5eac2b0a20fcad76cb715c21c8c5daba5694 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:10:55 -0400
Subject: [PATCH 04/59] Additional testing.

---
 src/allmydata/test/test_codec.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_codec.py b/src/allmydata/test/test_codec.py
index 3076a322e..ee64e2bf2 100644
--- a/src/allmydata/test/test_codec.py
+++ b/src/allmydata/test/test_codec.py
@@ -15,7 +15,7 @@ if PY2:
 import os
 from twisted.trial import unittest
 from twisted.python import log
-from allmydata.codec import CRSEncoder, CRSDecoder
+from allmydata.codec import CRSEncoder, CRSDecoder, parse_params
 import random
 from allmydata.util import mathutil
 
@@ -26,6 +26,8 @@ class T(unittest.TestCase):
         enc.set_params(size, required_shares, max_shares)
         params = enc.get_params()
         assert params == (size, required_shares, max_shares)
+        serialized_params = enc.get_serialized_params()
+        self.assertEqual(parse_params(serialized_params), params)
         log.msg("params: %s" % (params,))
         d = enc.encode(data0s)
         def _done_encoding_all(shares_and_shareids):

From 017104fec18339fe852e23e64f5b6503283c2fee Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:11:24 -0400
Subject: [PATCH 05/59] News file.

---
 newsfragments/3374.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3374.minor

diff --git a/newsfragments/3374.minor b/newsfragments/3374.minor
new file mode 100644
index 000000000..e69de29bb

From b82e2ad1c507c1b0e3dbf3fdfd53d2ee5e2ccddc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:26:44 -0400
Subject: [PATCH 06/59] Tests for allmydata.monitor.

---
 src/allmydata/test/test_monitor.py | 52 ++++++++++++++++++++++++++++++
 src/allmydata/util/_python3.py     |  1 +
 2 files changed, 53 insertions(+)
 create mode 100644 src/allmydata/test/test_monitor.py

diff --git a/src/allmydata/test/test_monitor.py b/src/allmydata/test/test_monitor.py
new file mode 100644
index 000000000..7010da73a
--- /dev/null
+++ b/src/allmydata/test/test_monitor.py
@@ -0,0 +1,52 @@
+"""
+Tests for allmydata.monitor.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from twisted.trial import unittest
+
+from allmydata.monitor import Monitor, OperationCancelledError
+
+
+class MonitorTests(unittest.TestCase):
+    """Tests for the Monitor class."""
+
+    def test_cancellation(self):
+        """The monitor can be cancelled."""
+        m = Monitor()
+        self.assertFalse(m.is_cancelled())
+        m.raise_if_cancelled()
+        m.cancel()
+        self.assertTrue(m.is_cancelled())
+        with self.assertRaises(OperationCancelledError):
+            m.raise_if_cancelled()
+
+    def test_status(self):
+        """The monitor can have its status set."""
+        m = Monitor()
+        self.assertEqual(m.get_status(), None)
+        m.set_status("discombobulated")
+        self.assertEqual(m.get_status(), "discombobulated")
+
+    def test_finish(self):
+        """The monitor can finish."""
+        m = Monitor()
+        self.assertFalse(m.is_finished())
+        d = m.when_done()
+        self.assertNoResult(d)
+
+        result = m.finish(300)
+        self.assertEqual(result, 300)
+        self.assertEqual(m.get_status(), 300)
+        self.assertTrue(m.is_finished())
+
+        d.addBoth(self.assertEqual, 300)
+        return d
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 2b18eaa1d..f8260078f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -61,6 +61,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_humanreadable",
     "allmydata.test.test_iputil",
     "allmydata.test.test_log",
+    "allmydata.test.test_monitor",
     "allmydata.test.test_netstring",
     "allmydata.test.test_observer",
     "allmydata.test.test_pipeline",

From 0e034e06b771782b081986d12bd9dc066f3f3546 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:28:14 -0400
Subject: [PATCH 07/59] Port to Python 3.

---
 src/allmydata/monitor.py       | 14 ++++++++++++++
 src/allmydata/util/_python3.py |  1 +
 2 files changed, 15 insertions(+)

diff --git a/src/allmydata/monitor.py b/src/allmydata/monitor.py
index 98c463014..1559a30d9 100644
--- a/src/allmydata/monitor.py
+++ b/src/allmydata/monitor.py
@@ -1,7 +1,21 @@
+"""
+Manage status of long-running operations.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from zope.interface import Interface, implementer
 from allmydata.util import observer
 
+
 class IMonitor(Interface):
     """I manage status, progress, and cancellation for long-running operations.
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f8260078f..f05beed88 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -24,6 +24,7 @@ PORTED_MODULES = [
     "allmydata.crypto.util",
     "allmydata.hashtree",
     "allmydata.interfaces",
+    "allmydata.monitor",
     "allmydata.test.common_py3",
     "allmydata.util._python3",
     "allmydata.util.abbreviate",

From 6229bb4e04964fe5347eb79231cf43537367d460 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:29:30 -0400
Subject: [PATCH 08/59] Ratchet.

---
 misc/python3/ratchet-passing | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bbc6e6c7e..f5394e965 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -24,6 +24,9 @@ allmydata.test.test_base62.Base62.test_known_values
 allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
 allmydata.test.test_base62.Base62.test_odd_sizes
 allmydata.test.test_base62.Base62.test_roundtrip
+allmydata.test.test_codec.T.test_encode
+allmydata.test.test_codec.T.test_encode1
+allmydata.test.test_codec.T.test_encode2
 allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_key_serialization
@@ -91,6 +94,9 @@ allmydata.test.test_log.Log.test_numming
 allmydata.test.test_log.Log.test_parent_id
 allmydata.test.test_log.Log.test_with_bytes_prefix
 allmydata.test.test_log.Log.test_with_prefix
+allmydata.test.test_monitor.MonitorTests.test_cancellation
+allmydata.test.test_monitor.MonitorTests.test_finish
+allmydata.test.test_monitor.MonitorTests.test_status
 allmydata.test.test_netstring.Netstring.test_encode
 allmydata.test.test_netstring.Netstring.test_extra
 allmydata.test.test_netstring.Netstring.test_nested

From 0a2b797c4964d51a38cbc75718f80e7938868336 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 13:32:12 -0400
Subject: [PATCH 09/59] Lint fix.

---
 src/allmydata/interfaces.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 7b02ceea9..5126b8c90 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -10,8 +10,9 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    # Don't import object/str/dict/etc. types, so we don't break any interfaces.
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, max, min  # noqa: F401
+    # Don't import object/str/dict/etc. types, so we don't break any
+    # interfaces. Not importing open() because it triggers bogus flake8 error.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min  # noqa: F401
 
 from past.builtins import long
 

From 24772616c2125806608279b80145f287619941bd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 7 Aug 2020 14:13:20 -0400
Subject: [PATCH 10/59] Bytes not strings.

---
 src/allmydata/interfaces.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 5126b8c90..e9e5a5eac 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -87,7 +87,7 @@ class RIBucketReader(RemoteInterface):
         """
 
 
-TestVector = ListOf(TupleOf(Offset, ReadSize, str, str))
+TestVector = ListOf(TupleOf(Offset, ReadSize, bytes, bytes))
 # elements are (offset, length, operator, specimen)
 # operator is one of "lt, le, eq, ne, ge, gt"
 # nop always passes and is used to fetch data while writing.
@@ -111,7 +111,7 @@ class RIStorageServer(RemoteInterface):
         """
         Return a dictionary of version information.
         """
-        return DictOf(str, Any())
+        return DictOf(bytes, Any())
 
     def allocate_buckets(storage_index=StorageIndex,
                          renew_secret=LeaseRenewSecret,

From 1c9e4ec842ccb42f670733071e2ac9ed869917fc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:04:52 -0400
Subject: [PATCH 11/59] Move connection_status tests into their own module.

---
 src/allmydata/test/test_connection_status.py | 112 +++++++++++++++++++
 src/allmydata/test/test_connections.py       | 104 -----------------
 2 files changed, 112 insertions(+), 104 deletions(-)
 create mode 100644 src/allmydata/test/test_connection_status.py

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
new file mode 100644
index 000000000..763c14c57
--- /dev/null
+++ b/src/allmydata/test/test_connection_status.py
@@ -0,0 +1,112 @@
+"""
+Tests for allmydata.util.connection_status.
+"""
+
+import mock
+
+from twisted.trial import unittest
+
+from ..util import connection_status
+
+class Status(unittest.TestCase):
+    def test_hint_statuses(self):
+        ncs = connection_status._hint_statuses(["h2","h1"],
+                                               {"h1": "hand1", "h4": "hand4"},
+                                               {"h1": "st1", "h2": "st2",
+                                                "h3": "st3"})
+        self.assertEqual(ncs, {"h1 via hand1": "st1",
+                               "h2": "st2"})
+
+    def test_reconnector_connected(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_others(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_listener(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.listenerStatus = ("listener1", "successful")
+        ci.winningHint = None
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected via listener (listener1)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connecting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "connecting"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary, "Trying to connect")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_waiting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "waiting"
+        ri.lastAttempt = 10
+        ri.nextAttempt = 20
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        with mock.patch("time.time", return_value=12):
+            cs = connection_status.from_foolscap_reconnector(rc, 5)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary,
+                         "Reconnecting in 8 seconds (last attempt 2s ago)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 5)
diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py
index 3e2806dd0..9b5bd7f30 100644
--- a/src/allmydata/test/test_connections.py
+++ b/src/allmydata/test/test_connections.py
@@ -7,7 +7,6 @@ from foolscap.connections import tcp
 from ..node import PrivacyError, config_from_string
 from ..node import create_connection_handlers
 from ..node import create_main_tub, _tub_portlocation
-from ..util import connection_status
 from ..util.i2p_provider import create as create_i2p_provider
 from ..util.tor_provider import create as create_tor_provider
 
@@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
             str(ctx.exception),
             "tub.location includes tcp: hint",
         )
-
-class Status(unittest.TestCase):
-    def test_hint_statuses(self):
-        ncs = connection_status._hint_statuses(["h2","h1"],
-                                               {"h1": "hand1", "h4": "hand4"},
-                                               {"h1": "st1", "h2": "st2",
-                                                "h3": "st3"})
-        self.assertEqual(ncs, {"h1 via hand1": "st1",
-                               "h2": "st2"})
-
-    def test_reconnector_connected(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_others(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_listener(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.listenerStatus = ("listener1", "successful")
-        ci.winningHint = None
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected via listener (listener1)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connecting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "connecting"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary, "Trying to connect")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_waiting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "waiting"
-        ri.lastAttempt = 10
-        ri.nextAttempt = 20
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        with mock.patch("time.time", return_value=12):
-            cs = connection_status.from_foolscap_reconnector(rc, 5)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary,
-                         "Reconnecting in 8 seconds (last attempt 2s ago)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 5)

From 243d02ecb1a01216fc16331ec25a04186dfce063 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:08:51 -0400
Subject: [PATCH 12/59] Port to Python 3.

---
 src/allmydata/test/test_connection_status.py | 10 ++++++++++
 src/allmydata/util/_python3.py               |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 763c14c57..2bd8bf6ab 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -1,6 +1,16 @@
 """
 Tests for allmydata.util.connection_status.
+
+Port to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import mock
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 091c248af..f074927b6 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -50,6 +50,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",

From a08cde9a4da3bf636560fc72e2297a371953b38e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 14:30:16 -0400
Subject: [PATCH 13/59] Port to Python 3.

---
 src/allmydata/interfaces.py             |  1 +
 src/allmydata/util/_python3.py          |  1 +
 src/allmydata/util/connection_status.py | 22 ++++++++++++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 36eb55bfb..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,3 +1,4 @@
+from past.builtins import long
 
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f074927b6..494d5e161 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
     "allmydata.util.gcutil",
diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index 44c12f220..0e8595e81 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -1,3 +1,18 @@
+"""
+Parse connection status from Foolscap.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus
@@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
 
 def from_foolscap_reconnector(rc, last_received):
     ri = rc.getReconnectionInfo()
-    # See foolscap/reconnector.py, ReconnectionInfo, for details about
-    # possible states.
+    # See foolscap/reconnector.py, ReconnectionInfo, for details about possible
+    # states. The returned result is a native string, it seems, so convert to
+    # unicode.
     state = ri.state
+    if isinstance(state, bytes):  # Python 2
+        state = str(state, "ascii")
     if state == "unstarted":
         return ConnectionStatus.unstarted()
 

From e24c21bef779de6d1d419b2135231f628a0bcc20 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:38:58 -0400
Subject: [PATCH 14/59] Make configutil tests more standalone, and less
 repetitive.

---
 src/allmydata/test/cli/test_create.py | 27 ++++++++
 src/allmydata/test/test_configutil.py | 90 +++++++--------------------
 2 files changed, 49 insertions(+), 68 deletions(-)

diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py
index 75162c39e..f013c0205 100644
--- a/src/allmydata/test/cli/test_create.py
+++ b/src/allmydata/test/cli/test_create.py
@@ -6,6 +6,8 @@ from twisted.python import usage
 from allmydata.util import configutil
 from ..common_util import run_cli, parse_cli
 from ...scripts import create_node
+from ... import client
+
 
 def read_config(basedir):
     tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@@ -33,6 +35,31 @@ class Config(unittest.TestCase):
             e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
             self.assertIn("option %s not recognized" % (option,), str(e))
 
+    def test_create_client_config(self):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+
+        with open(fname, 'w') as f:
+            opts = {"nickname": "nick",
+                    "webport": "tcp:3456",
+                    "hide-ip": False,
+                    "listen": "none",
+                    "shares-needed": "1",
+                    "shares-happy": "1",
+                    "shares-total": "1",
+                    }
+            create_node.write_node_config(f, opts)
+            create_node.write_client_config(f, opts)
+
+        config = configutil.get_config(fname)
+        # should succeed, no exceptions
+        configutil.validate_config(
+            fname,
+            config,
+            client._valid_config(),
+        )
+
     @defer.inlineCallbacks
     def test_client(self):
         basedir = self.mktemp()
diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index 45eb6ac25..ffb5f5320 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -3,12 +3,9 @@ import os.path
 from twisted.trial import unittest
 
 from allmydata.util import configutil
-from allmydata.test.no_network import GridTestMixin
-from ..scripts import create_node
-from .. import client
 
 
-class ConfigUtilTests(GridTestMixin, unittest.TestCase):
+class ConfigUtilTests(unittest.TestCase):
     def setUp(self):
         super(ConfigUtilTests, self).setUp()
         self.static_valid_config = configutil.ValidConfiguration(
@@ -20,10 +17,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
         )
 
+    def create_tahoe_cfg(self, cfg):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+        with open(fname, "w") as f:
+            f.write(cfg)
+        return fname
+
     def test_config_utils(self):
-        self.basedir = "cli/ConfigUtilTests/test-config-utils"
-        self.set_up_grid(oneshare=True)
-        tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
+        tahoe_cfg = self.create_tahoe_cfg("""\
+[node]
+nickname = client-0
+web.port = adopt-socket:fd=5
+[storage]
+enabled = false
+""")
 
         # test that at least one option was read correctly
         config = configutil.get_config(tahoe_cfg)
@@ -45,12 +54,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
 
     def test_config_validation_success(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -66,12 +70,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         validation but are matched by the dynamic validation is considered
         valid.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -82,12 +81,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         )
 
     def test_config_validation_invalid_item(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -103,12 +97,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -124,12 +113,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -145,12 +129,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section, item pair that is matched by neither the
         static nor dynamic validators is rejected.
         """
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -160,28 +139,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             self.dynamic_valid_config,
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
-
-    def test_create_client_config(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            opts = {"nickname": "nick",
-                    "webport": "tcp:3456",
-                    "hide-ip": False,
-                    "listen": "none",
-                    "shares-needed": "1",
-                    "shares-happy": "1",
-                    "shares-total": "1",
-                    }
-            create_node.write_node_config(f, opts)
-            create_node.write_client_config(f, opts)
-
-        config = configutil.get_config(fname)
-        # should succeed, no exceptions
-        configutil.validate_config(
-            fname,
-            config,
-            client._valid_config(),
-        )

From babe2dbc85d8c6d87bec5e1a919faf71f854b596 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:45:52 -0400
Subject: [PATCH 15/59] Port to Python 3.

---
 src/allmydata/test/test_configutil.py | 15 +++++++++++++++
 src/allmydata/util/_python3.py        |  1 +
 2 files changed, 16 insertions(+)

diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py
index ffb5f5320..c57381289 100644
--- a/src/allmydata/test/test_configutil.py
+++ b/src/allmydata/test/test_configutil.py
@@ -1,3 +1,18 @@
+"""
+Tests for allmydata.util.configutil.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Omitted dict, cause worried about interactions.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 
 from twisted.trial import unittest
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 494d5e161..b6c4b6c87 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -51,6 +51,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_configutil",
     "allmydata.test.test_connection_status",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",

From 11b934120c1b5f279e4c84c1d94de8bec9ea2bc8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:49:59 -0400
Subject: [PATCH 16/59] Port to Python 3.

---
 src/allmydata/util/_python3.py   |  1 +
 src/allmydata/util/configutil.py | 18 +++++++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index b6c4b6c87..947fcf00b 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -28,6 +28,7 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.configutil",
     "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.dictutil",
diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 3699db35d..295605b65 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,5 +1,13 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
-from ConfigParser import SafeConfigParser
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from configparser import SafeConfigParser
 
 import attr
 
@@ -13,11 +21,7 @@ class UnknownConfigError(Exception):
 
 def get_config(tahoe_cfg):
     config = SafeConfigParser()
-    with open(tahoe_cfg, "rb") as f:
-        # Skip any initial Byte Order Mark. Since this is an ordinary file, we
-        # don't need to handle incomplete reads, and can assume seekability.
-        if f.read(3) != '\xEF\xBB\xBF':
-            f.seek(0)
+    with open(tahoe_cfg, "r") as f:
         config.readfp(f)
     return config
 
@@ -28,7 +32,7 @@ def set_config(config, section, option, value):
     assert config.get(section, option) == value
 
 def write_config(tahoe_cfg, config):
-    with open(tahoe_cfg, "wb") as f:
+    with open(tahoe_cfg, "w") as f:
         config.write(f)
 
 def validate_config(fname, cfg, valid_config):

From 277298050f16ab3794868f377b6a617820baefe7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 15:51:27 -0400
Subject: [PATCH 17/59] News file and ratchet.

---
 misc/python3/ratchet-passing | 13 +++++++++++++
 newsfragments/3377.minor     |  0
 2 files changed, 13 insertions(+)
 create mode 100644 newsfragments/3377.minor

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bbc6e6c7e..e6b0d60c2 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -24,6 +24,19 @@ allmydata.test.test_base62.Base62.test_known_values
 allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars
 allmydata.test.test_base62.Base62.test_odd_sizes
 allmydata.test.test_base62.Base62.test_roundtrip
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_dynamic_validation_success
+allmydata.test.test_configutil.ConfigUtilTests.test_config_utils
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_item
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_invalid_section
+allmydata.test.test_configutil.ConfigUtilTests.test_config_validation_success
+allmydata.test.test_connection_status.Status.test_hint_statuses
+allmydata.test.test_connection_status.Status.test_reconnector_connected
+allmydata.test.test_connection_status.Status.test_reconnector_connected_listener
+allmydata.test.test_connection_status.Status.test_reconnector_connected_others
+allmydata.test.test_connection_status.Status.test_reconnector_connecting
+allmydata.test.test_connection_status.Status.test_reconnector_waiting
 allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes
 allmydata.test.test_crypto.TestEd25519.test_key_serialization
diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor
new file mode 100644
index 000000000..e69de29bb

From 10378541d72a6853b4cc1f3c86482979f14b3827 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 11 Aug 2020 16:40:00 -0400
Subject: [PATCH 18/59] Use Python 2 ConfigParser, so correct exceptions get
 raised.

---
 src/allmydata/util/configutil.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 295605b65..79980b006 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -7,10 +7,17 @@ from future.utils import PY2
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from configparser import SafeConfigParser
+if PY2:
+    # In theory on Python 2 configparser also works, but then code gets the
+    # wrong exceptions and they don't get handled. So just use native parser
+    # for now.
+    from ConfigParser import SafeConfigParser
+else:
+    from configparser import SafeConfigParser
 
 import attr
 
+
 class UnknownConfigError(Exception):
     """
     An unknown config item was found.

From af61571fa61616e4bdd43326919075d46eeaf6cc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 15:56:08 -0400
Subject: [PATCH 19/59] Accidentally passing test :(

---
 misc/python3/ratchet-passing | 1 +
 1 file changed, 1 insertion(+)

diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing
index bf56e0de2..01e58a2fa 100644
--- a/misc/python3/ratchet-passing
+++ b/misc/python3/ratchet-passing
@@ -206,6 +206,7 @@ allmydata.test.test_statistics.Statistics.test_pr_file_loss
 allmydata.test.test_statistics.Statistics.test_repair_cost
 allmydata.test.test_statistics.Statistics.test_repair_count_pmf
 allmydata.test.test_statistics.Statistics.test_survival_pmf
+allmydata.test.test_stats.CPUUsage.test_monitor
 allmydata.test.test_time_format.TimeFormat.test_epoch
 allmydata.test.test_time_format.TimeFormat.test_epoch_in_London
 allmydata.test.test_time_format.TimeFormat.test_format_delta

From b0c4f6d2abaea2d2a2ca5408072cb8ba68fb827a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 13 Aug 2020 16:30:27 -0400
Subject: [PATCH 20/59] Fix Python 2 tests.

---
 src/allmydata/util/configutil.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py
index 79980b006..1a1a93f18 100644
--- a/src/allmydata/util/configutil.py
+++ b/src/allmydata/util/configutil.py
@@ -1,3 +1,10 @@
+"""
+Read/write config files.
+
+Configuration is returned as native strings.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -5,7 +12,9 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # We don't do open(), because we want files to read/write native strs when
+    # we do "r" or "w".
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 if PY2:
     # In theory on Python 2 configparser also works, but then code gets the
@@ -27,8 +36,17 @@ class UnknownConfigError(Exception):
 
 
 def get_config(tahoe_cfg):
+    """Load the config, returning a SafeConfigParser.
+
+    Configuration is returned as native strings.
+    """
     config = SafeConfigParser()
     with open(tahoe_cfg, "r") as f:
+        # On Python 2, where we read in bytes, skip any initial Byte Order
+        # Mark. Since this is an ordinary file, we don't need to handle
+        # incomplete reads, and can assume seekability.
+        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
+            f.seek(0)
         config.readfp(f)
     return config
 

From fa36fb9ab9ff0f07848fe4112cb85beb07473292 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Sat, 22 Aug 2020 16:46:03 -0400
Subject: [PATCH 21/59] Remove unused
 allmydata.test.common_web.WebRenderingMixin

---
 newsfragments/3392.minor         |  0
 src/allmydata/test/common_web.py | 57 --------------------------------
 2 files changed, 57 deletions(-)
 create mode 100644 newsfragments/3392.minor

diff --git a/newsfragments/3392.minor b/newsfragments/3392.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py
index 791a8d9ab..1c9312689 100644
--- a/src/allmydata/test/common_web.py
+++ b/src/allmydata/test/common_web.py
@@ -1,64 +1,7 @@
 
-import re
 import treq
 from twisted.internet import defer
 from twisted.web.error import Error
-from nevow.testutil import FakeRequest
-from nevow import inevow, context
-
-class WebRenderingMixin(object):
-    # d=page.renderString() or s=page.renderSynchronously() will exercise
-    # docFactory, render_*/data_* . It won't exercise want_json(), or my
-    # renderHTTP() override which tests want_json(). To exercise args=, we
-    # must build a context. Pages which use a return_to= argument need a
-    # context.
-
-    # d=page.renderHTTP(ctx) will exercise my renderHTTP, want_json, and
-    # docFactory/render_*/data_*, but it requires building a context. Since
-    # we're already building a context, it is easy to exercise args= .
-
-    # so, use at least two d=page.renderHTTP(ctx) per page (one for json, one
-    # for html), then use lots of simple s=page.renderSynchronously() to
-    # exercise the fine details (the ones that don't require args=).
-
-    def make_context(self, req):
-        ctx = context.RequestContext(tag=req)
-        ctx.remember(req, inevow.IRequest)
-        ctx.remember(None, inevow.IData)
-        ctx = context.WovenContext(parent=ctx, precompile=False)
-        return ctx
-
-    def render1(self, page, **kwargs):
-        # use this to exercise an overridden renderHTTP, usually for
-        # output=json or render_GET. It always returns a Deferred.
-        req = FakeRequest(**kwargs)
-        req.fields = None
-        ctx = self.make_context(req)
-        d = defer.maybeDeferred(page.renderHTTP, ctx)
-        def _done(res):
-            if isinstance(res, str):
-                return res + req.v
-            return req.v
-        d.addCallback(_done)
-        return d
-
-    def render2(self, page, **kwargs):
-        # use this to exercise the normal Nevow docFactory rendering. It
-        # returns a string. If one of the render_* methods returns a
-        # Deferred, this will throw an exception. (note that
-        # page.renderString is the Deferred-returning equivalent)
-        req = FakeRequest(**kwargs)
-        req.fields = None
-        ctx = self.make_context(req)
-        return page.renderSynchronously(ctx)
-
-    def failUnlessIn(self, substring, s):
-        self.failUnless(substring in s, s)
-
-    def remove_tags(self, s):
-        s = re.sub(r'<[^>]*>', ' ', s)
-        s = re.sub(r'\s+', ' ', s)
-        return s
 
 @defer.inlineCallbacks
 def do_http(method, url, **kwargs):

From 57fdead1c37a9e8f11f1ec609ddecc3a1183129c Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Sat, 22 Aug 2020 17:21:24 -0400
Subject: [PATCH 22/59] Remove allmydata.web._nevow_106

---
 newsfragments/3393.minor        |  0
 src/allmydata/web/_nevow_106.py | 31 -------------------------------
 src/allmydata/web/private.py    |  5 -----
 3 files changed, 36 deletions(-)
 create mode 100644 newsfragments/3393.minor
 delete mode 100644 src/allmydata/web/_nevow_106.py

diff --git a/newsfragments/3393.minor b/newsfragments/3393.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/web/_nevow_106.py b/src/allmydata/web/_nevow_106.py
deleted file mode 100644
index 3379767a5..000000000
--- a/src/allmydata/web/_nevow_106.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Implement a work-around for <https://github.com/twisted/nevow/issues/106>.
-"""
-
-from __future__ import (
-    print_function,
-    unicode_literals,
-    absolute_import,
-    division,
-)
-
-from nevow import inevow
-from twisted.internet import defer
-
-def renderHTTP(self, ctx):
-    request = inevow.IRequest(ctx)
-    if self.real_prepath_len is not None:
-        request.postpath = request.prepath + request.postpath
-        request.prepath = request.postpath[:self.real_prepath_len]
-        del request.postpath[:self.real_prepath_len]
-    result = defer.maybeDeferred(self.original.render, request).addCallback(
-        self._handle_NOT_DONE_YET, request)
-    return result
-
-
-def patch():
-    """
-    Monkey-patch the proposed fix into place.
-    """
-    from nevow.appserver import OldResourceAdapter
-    OldResourceAdapter.renderHTTP = renderHTTP
diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py
index fdf67a64f..fea058405 100644
--- a/src/allmydata/web/private.py
+++ b/src/allmydata/web/private.py
@@ -54,11 +54,6 @@ from .logs import (
     create_log_resources,
 )
 
-# Hotfix work-around https://github.com/twisted/nevow/issues/106
-from . import _nevow_106
-_nevow_106.patch()
-del _nevow_106
-
 SCHEME = b"tahoe-lafs"
 
 class IToken(ICredentials):

From 37ca3c5181bf08efd21be5e5483c40a7c0895297 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 25 Aug 2020 11:55:12 -0400
Subject: [PATCH 23/59] This should be bytes, as it's encoded as URI extension
 which still expects bytes.

---
 src/allmydata/codec.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py
index 048355871..a2bf18f02 100644
--- a/src/allmydata/codec.py
+++ b/src/allmydata/codec.py
@@ -22,7 +22,7 @@ import zfec
 
 @implementer(ICodecEncoder)
 class CRSEncoder(object):
-    ENCODER_TYPE = "crs"
+    ENCODER_TYPE = b"crs"
 
     def set_params(self, data_size, required_shares, max_shares):
         assert required_shares <= max_shares
@@ -40,8 +40,8 @@ class CRSEncoder(object):
         return (self.data_size, self.required_shares, self.max_shares)
 
     def get_serialized_params(self):
-        return native_str("%d-%d-%d" % (self.data_size, self.required_shares,
-                                        self.max_shares))
+        return b"%d-%d-%d" % (self.data_size, self.required_shares,
+                              self.max_shares)
 
     def get_block_size(self):
         return self.share_size
@@ -84,5 +84,5 @@ class CRSDecoder(object):
         return defer.succeed(data)
 
 def parse_params(serializedparams):
-    pieces = serializedparams.split("-")
+    pieces = serializedparams.split(b"-")
     return int(pieces[0]), int(pieces[1]), int(pieces[2])

From d5ba10544e278df521f18e28e424815853fc5d46 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 25 Aug 2020 12:08:59 -0400
Subject: [PATCH 24/59] Keep interfaces more backwards compatible.

---
 src/allmydata/interfaces.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 98ab7d271..43565dc4f 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -16,8 +16,6 @@ if PY2:
 
 from past.builtins import long
 
-from past.builtins import long
-
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
     IPlugin,
@@ -76,7 +74,7 @@ class RIBucketReader(RemoteInterface):
     def read(offset=Offset, length=ReadSize):
         return ShareData
 
-    def advise_corrupt_share(reason=str):
+    def advise_corrupt_share(reason=bytes):
         """Clients who discover hash failures in shares that they have
         downloaded from me will use this method to inform me about the
         failures. I will record their concern so that my operator can
@@ -107,7 +105,7 @@ ReadData = ListOf(ShareData)
 
 
 class RIStorageServer(RemoteInterface):
-    __remote_name__ = "RIStorageServer.tahoe.allmydata.com"
+    __remote_name__ = b"RIStorageServer.tahoe.allmydata.com"
 
     def get_version():
         """
@@ -295,8 +293,8 @@ class RIStorageServer(RemoteInterface):
         """
         return TupleOf(bool, DictOf(int, ReadData))
 
-    def advise_corrupt_share(share_type=str, storage_index=StorageIndex,
-                             shnum=int, reason=str):
+    def advise_corrupt_share(share_type=bytes, storage_index=StorageIndex,
+                             shnum=int, reason=bytes):
         """Clients who discover hash failures in shares that they have
         downloaded from me will use this method to inform me about the
         failures. I will record their concern so that my operator can
@@ -2877,7 +2875,7 @@ UploadResults = Any() #DictOf(str, str)
 
 
 class RIEncryptedUploadable(RemoteInterface):
-    __remote_name__ = "RIEncryptedUploadable.tahoe.allmydata.com"
+    __remote_name__ = b"RIEncryptedUploadable.tahoe.allmydata.com"
 
     def get_size():
         return Offset
@@ -2893,7 +2891,7 @@ class RIEncryptedUploadable(RemoteInterface):
 
 
 class RICHKUploadHelper(RemoteInterface):
-    __remote_name__ = "RIUploadHelper.tahoe.allmydata.com"
+    __remote_name__ = b"RIUploadHelper.tahoe.allmydata.com"
 
     def get_version():
         """
@@ -2906,7 +2904,7 @@ class RICHKUploadHelper(RemoteInterface):
 
 
 class RIHelper(RemoteInterface):
-    __remote_name__ = "RIHelper.tahoe.allmydata.com"
+    __remote_name__ = b"RIHelper.tahoe.allmydata.com"
 
     def get_version():
         """
@@ -2933,7 +2931,7 @@ class RIHelper(RemoteInterface):
 
 
 class RIStatsProvider(RemoteInterface):
-    __remote_name__ = "RIStatsProvider.tahoe.allmydata.com"
+    __remote_name__ = b"RIStatsProvider.tahoe.allmydata.com"
     """
     Provides access to statistics and monitoring information.
     """
@@ -2950,7 +2948,7 @@ class RIStatsProvider(RemoteInterface):
 
 
 class RIStatsGatherer(RemoteInterface):
-    __remote_name__ = "RIStatsGatherer.tahoe.allmydata.com"
+    __remote_name__ = b"RIStatsGatherer.tahoe.allmydata.com"
     """
     Provides a monitoring service for centralised collection of stats
     """

From 8682550961bb799399adb0571442ff5caf990d09 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:35:25 -0400
Subject: [PATCH 25/59] More passing tests on Python 3.

---
 src/allmydata/test/test_storage_web.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index 9c64c2f45..e8c5dc8a7 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -107,7 +107,6 @@ class MyStorageServer(StorageServer):
 
 class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -130,12 +129,12 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
 
         # this sample is before the crawler has started doing anything
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Reserved space: - 0 B (0)", s)
-        self.failUnlessIn("Total buckets: Not computed yet", s)
-        self.failUnlessIn("Next crawl in", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
+        self.failUnlessIn(b"Total buckets: Not computed yet", s)
+        self.failUnlessIn(b"Next crawl in", s)
 
         # give the bucket-counting-crawler one tick to get started. The
         # cpu_slice=0 will force it to yield right after it processes the
@@ -154,8 +153,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn(" Current crawl ", s)
-            self.failUnlessIn(" (next work in ", s)
+            self.failUnlessIn(b" Current crawl ", s)
+            self.failUnlessIn(b" (next work in ", s)
         d.addCallback(_check)
 
         # now give it enough time to complete a full cycle
@@ -166,8 +165,8 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             ss.bucket_counter.cpu_slice = orig_cpu_slice
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("Total buckets: 0 (the number of", s)
-            self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
+            self.failUnlessIn(b"Total buckets: 0 (the number of", s)
+            self.failUnless(b"Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
         d.addCallback(_check2)
         return d
 
@@ -228,20 +227,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
             # no ETA is available yet
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (next work", s)
+            self.failUnlessIn(b"complete (next work", s)
 
         def _check_2(ignored):
             # one prefix has finished, so an ETA based upon that elapsed time
             # should be available.
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
 
         def _check_3(ignored):
             # two prefixes have finished
             html = renderSynchronously(w)
             s = remove_tags(html)
-            self.failUnlessIn("complete (ETA ", s)
+            self.failUnlessIn(b"complete (ETA ", s)
             d.callback("done")
 
         hooks[0].addCallback(_check_1).addErrback(d.errback)

From 3d18b24967c43433902f8ffbc94b3569f67292b4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:15 -0400
Subject: [PATCH 26/59] Port even more tests to Python 3.

---
 src/allmydata/test/test_storage_web.py | 55 +++++++++++++-------------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index e8c5dc8a7..bface4b13 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -1171,7 +1171,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
 class WebStatus(unittest.TestCase, pollmixin.PollMixin):
 
-    @skipIf(PY3, "Not ported yet.")
     def setUp(self):
         self.s = service.MultiService()
         self.s.startService()
@@ -1181,7 +1180,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
     def test_no_server(self):
         w = StorageStatus(None)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
+        self.failUnlessIn(b"<h1>No Storage Server Running</h1>", html)
 
     def test_status(self):
         basedir = "storage/WebStatus/status"
@@ -1192,12 +1191,12 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss, "nickname")
         d = renderDeferred(w)
         def _check_html(html):
-            self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+            self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
             s = remove_tags(html)
-            self.failUnlessIn("Server Nickname: nickname", s)
-            self.failUnlessIn("Server Nodeid: %s"  % base32.b2a(nodeid), s)
-            self.failUnlessIn("Accepting new shares: Yes", s)
-            self.failUnlessIn("Reserved space: - 0 B (0)", s)
+            self.failUnlessIn(b"Server Nickname: nickname", s)
+            self.failUnlessIn(b"Server Nodeid: %s"  % base32.b2a(nodeid), s)
+            self.failUnlessIn(b"Accepting new shares: Yes", s)
+            self.failUnlessIn(b"Reserved space: - 0 B (0)", s)
         d.addCallback(_check_html)
         d.addCallback(lambda ign: renderJSON(w))
         def _check_json(raw):
@@ -1224,11 +1223,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: Yes", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: Yes", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnless(ss.get_available_space() is None)
 
     def test_status_bad_disk_stats(self):
@@ -1244,11 +1243,11 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
-        self.failUnlessIn("Total disk space: ?", s)
-        self.failUnlessIn("Space Available to Tahoe: ?", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
+        self.failUnlessIn(b"Total disk space: ?", s)
+        self.failUnlessIn(b"Space Available to Tahoe: ?", s)
         self.failUnlessEqual(ss.get_available_space(), 0)
 
     def test_status_right_disk_stats(self):
@@ -1281,14 +1280,14 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         w = StorageStatus(ss)
         html = renderSynchronously(w)
 
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Total disk space: 5.00 GB", s)
-        self.failUnlessIn("Disk space used: - 1.00 GB", s)
-        self.failUnlessIn("Disk space free (root): 4.00 GB", s)
-        self.failUnlessIn("Disk space free (non-root): 3.00 GB", s)
-        self.failUnlessIn("Reserved space: - 1.00 GB", s)
-        self.failUnlessIn("Space Available to Tahoe: 2.00 GB", s)
+        self.failUnlessIn(b"Total disk space: 5.00 GB", s)
+        self.failUnlessIn(b"Disk space used: - 1.00 GB", s)
+        self.failUnlessIn(b"Disk space free (root): 4.00 GB", s)
+        self.failUnlessIn(b"Disk space free (non-root): 3.00 GB", s)
+        self.failUnlessIn(b"Reserved space: - 1.00 GB", s)
+        self.failUnlessIn(b"Space Available to Tahoe: 2.00 GB", s)
         self.failUnlessEqual(ss.get_available_space(), 2*GB)
 
     def test_readonly(self):
@@ -1298,9 +1297,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Accepting new shares: No", s)
+        self.failUnlessIn(b"Accepting new shares: No", s)
 
     def test_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1309,9 +1308,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_huge_reserved(self):
         basedir = "storage/WebStatus/reserved"
@@ -1320,9 +1319,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin):
         ss.setServiceParent(self.s)
         w = StorageStatus(ss)
         html = renderSynchronously(w)
-        self.failUnlessIn("<h1>Storage Server Status</h1>", html)
+        self.failUnlessIn(b"<h1>Storage Server Status</h1>", html)
         s = remove_tags(html)
-        self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
+        self.failUnlessIn(b"Reserved space: - 10.00 MB (10000000)", s)
 
     def test_util(self):
         w = StorageStatusElement(None, None)

From 431e939bb82533944aa22890d969d7ca3fca3337 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:38:52 -0400
Subject: [PATCH 27/59] Finish porting test_storage_web to Python 3.

---
 newsfragments/3395.minor       | 0
 src/allmydata/util/_python3.py | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 newsfragments/3395.minor

diff --git a/newsfragments/3395.minor b/newsfragments/3395.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 6aa1010bc..82ddebc72 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -83,7 +83,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
-    "allmydata.test.test_storage_web",  # partial, WIP
+    "allmydata.test.test_storage_web",
     "allmydata.test.test_time_format",
     "allmydata.test.test_uri",
     "allmydata.test.test_util",

From a758f32edfd816ca4d56e797767fdaa4312f5bb1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:53:02 -0400
Subject: [PATCH 28/59] Try to make test_storage import on Python 3.

---
 src/allmydata/client.py            |  5 ++-
 src/allmydata/immutable/upload.py  |  4 ++-
 src/allmydata/node.py              |  5 ++-
 src/allmydata/storage_client.py    | 15 +++++----
 src/allmydata/test/common.py       | 47 ++--------------------------
 src/allmydata/test/common_py3.py   | 49 ++++++++++++++++++++++++++++++
 src/allmydata/test/test_storage.py |  3 +-
 7 files changed, 72 insertions(+), 56 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index f731c3163..845290ac0 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -2,7 +2,10 @@ import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
 from errno import ENOENT, EPERM
-from ConfigParser import NoSectionError
+try:
+    from ConfigParser import NoSectionError
+except ImportError:
+    from configparser import NoSectionError
 
 from foolscap.furl import (
     decode_furl,
diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index fe77fdf69..58cbea2ef 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -1,3 +1,5 @@
+from past.builtins import long
+
 import os, time, weakref, itertools
 from zope.interface import implementer
 from twisted.python import failure
@@ -26,7 +28,7 @@ from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \
 from allmydata.immutable import layout
 
 from six.moves import cStringIO as StringIO
-from happiness_upload import share_placement, calculate_happiness
+from .happiness_upload import share_placement, calculate_happiness
 
 from ..util.eliotutil import (
     log_call_deferred,
diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 6b3911d95..95033d36c 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -7,7 +7,10 @@ import os.path
 import re
 import types
 import errno
-import ConfigParser
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
 import tempfile
 from io import BytesIO
 from base64 import b32decode, b32encode
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 2f1823058..cfc3bc83f 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -30,9 +30,12 @@ the foolscap-based server implemented in src/allmydata/storage/*.py .
 
 
 import re, time, hashlib
-from ConfigParser import (
-    NoSectionError,
-)
+try:
+    from ConfigParser import (
+        NoSectionError,
+    )
+except ImportError:
+    from configparser import NoSectionError
 import attr
 from zope.interface import (
     Attribute,
@@ -534,11 +537,11 @@ class _NullStorage(object):
     which we can't communicate.
     """
     nickname = ""
-    permutation_seed = hashlib.sha256("").digest()
-    tubid = hashlib.sha256("").digest()
+    permutation_seed = hashlib.sha256(b"").digest()
+    tubid = hashlib.sha256(b"").digest()
     storage_server = None
 
-    lease_seed = hashlib.sha256("").digest()
+    lease_seed = hashlib.sha256(b"").digest()
 
     name = "<unsupported>"
     longname = "<storage with unsupported protocol>"
diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index a2af857b9..dbd93df67 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -88,6 +88,8 @@ from ..crypto import (
 from .eliotutil import (
     EliotLoggedRunTest,
 )
+# Backwards compatibility imports:
+from .common_py3 import LoggingServiceParent, ShouldFailMixin  # noqa: F401
 
 
 TEST_RSA_KEY_SIZE = 522
@@ -780,53 +782,8 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
     return filenode
 
 
-class LoggingServiceParent(service.MultiService):
-    def log(self, *args, **kwargs):
-        return log.msg(*args, **kwargs)
-
-
 TEST_DATA="\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)
 
-class ShouldFailMixin(object):
-    def shouldFail(self, expected_failure, which, substring,
-                   callable, *args, **kwargs):
-        """Assert that a function call raises some exception. This is a
-        Deferred-friendly version of TestCase.assertRaises() .
-
-        Suppose you want to verify the following function:
-
-         def broken(a, b, c):
-             if a < 0:
-                 raise TypeError('a must not be negative')
-             return defer.succeed(b+c)
-
-        You can use:
-            d = self.shouldFail(TypeError, 'test name',
-                                'a must not be negative',
-                                broken, -4, 5, c=12)
-        in your test method. The 'test name' string will be included in the
-        error message, if any, because Deferred chains frequently make it
-        difficult to tell which assertion was tripped.
-
-        The substring= argument, if not None, must appear in the 'repr'
-        of the message wrapped by this Failure, or the test will fail.
-        """
-
-        assert substring is None or isinstance(substring, str)
-        d = defer.maybeDeferred(callable, *args, **kwargs)
-        def done(res):
-            if isinstance(res, failure.Failure):
-                res.trap(expected_failure)
-                if substring:
-                    message = repr(res.value.args[0])
-                    self.failUnless(substring in message,
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, message))
-            else:
-                self.fail("%s was supposed to raise %s, not get '%s'" %
-                          (which, expected_failure, res))
-        d.addBoth(done)
-        return d
 
 class WebErrorMixin(object):
     def explain_web_error(self, f):
diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py
index 0daf66e62..36738d559 100644
--- a/src/allmydata/test/common_py3.py
+++ b/src/allmydata/test/common_py3.py
@@ -19,11 +19,13 @@ import time
 import signal
 
 from twisted.internet import defer, reactor
+from twisted.application import service
 from twisted.python import failure
 from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..util.encodingutil import unicode_platform, get_filesystem_encoding
+from ..util import log
 
 
 class TimezoneMixin(object):
@@ -135,3 +137,50 @@ class FakeCanary(object):
         if self.ignore:
             return
         del self.disconnectors[marker]
+
+
+class LoggingServiceParent(service.MultiService):
+    def log(self, *args, **kwargs):
+        return log.msg(*args, **kwargs)
+
+
+class ShouldFailMixin(object):
+    def shouldFail(self, expected_failure, which, substring,
+                   callable, *args, **kwargs):
+        """Assert that a function call raises some exception. This is a
+        Deferred-friendly version of TestCase.assertRaises() .
+
+        Suppose you want to verify the following function:
+
+         def broken(a, b, c):
+             if a < 0:
+                 raise TypeError('a must not be negative')
+             return defer.succeed(b+c)
+
+        You can use:
+            d = self.shouldFail(TypeError, 'test name',
+                                'a must not be negative',
+                                broken, -4, 5, c=12)
+        in your test method. The 'test name' string will be included in the
+        error message, if any, because Deferred chains frequently make it
+        difficult to tell which assertion was tripped.
+
+        The substring= argument, if not None, must appear in the 'repr'
+        of the message wrapped by this Failure, or the test will fail.
+        """
+
+        assert substring is None or isinstance(substring, str)
+        d = defer.maybeDeferred(callable, *args, **kwargs)
+        def done(res):
+            if isinstance(res, failure.Failure):
+                res.trap(expected_failure)
+                if substring:
+                    message = repr(res.value.args[0])
+                    self.failUnless(substring in message,
+                                    "%s: substring '%s' not in '%s'"
+                                    % (which, substring, message))
+            else:
+                self.fail("%s was supposed to raise %s, not get '%s'" %
+                          (which, expected_failure, res))
+        d.addBoth(done)
+        return d
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 14c342c41..d04e6b83d 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -30,12 +30,11 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \
                                      VERIFICATION_KEY_SIZE, \
                                      SHARE_HASH_CHAIN_SIZE
 from allmydata.interfaces import BadWriteEnablerError
-from allmydata.test.common import LoggingServiceParent, ShouldFailMixin
 from allmydata.test.no_network import NoNetworkServer
 from allmydata.storage_client import (
     _StorageServer,
 )
-from .common_py3 import FakeCanary
+from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
 
 
 class FakeStatsProvider(object):

From 637e8a054436d1df486f427a6dd1a5b097f24569 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 10:59:10 -0400
Subject: [PATCH 29/59] Remove duplication.

---
 src/allmydata/interfaces.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 298c826b7..c93c1d81d 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1,7 +1,5 @@
 from past.builtins import long
 
-from past.builtins import long
-
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
     IPlugin,

From b3460dcddc61e4b838239e8a2137ef6182f59327 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 11:00:19 -0400
Subject: [PATCH 30/59] Fix lint.

---
 src/allmydata/codec.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py
index a2bf18f02..a4baab4b6 100644
--- a/src/allmydata/codec.py
+++ b/src/allmydata/codec.py
@@ -11,7 +11,6 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-from future.utils import native_str
 
 from zope.interface import implementer
 from twisted.internet import defer

From 36177574be07e0ad081fbc2273bd9cef9b780525 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 26 Aug 2020 11:01:04 -0400
Subject: [PATCH 31/59] Fix lint.

---
 src/allmydata/test/test_storage_web.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_web.py b/src/allmydata/test/test_storage_web.py
index bface4b13..19f98851f 100644
--- a/src/allmydata/test/test_storage_web.py
+++ b/src/allmydata/test/test_storage_web.py
@@ -9,7 +9,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import PY2, PY3
+from future.utils import PY2
 if PY2:
     # Omitted list sinc it broke a test on Python 2. Shouldn't require further
     # work, when we switch to Python 3 we'll be dropping this, anyway.
@@ -19,7 +19,6 @@ import time
 import os.path
 import re
 import json
-from unittest import skipIf
 
 from twisted.trial import unittest
 

From 44126f840c71c21e7e797cc1de37cedb4d1464c7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:36:09 -0400
Subject: [PATCH 32/59] Refactor tox config to reduce duplication between
 py2/py3/coverage

The following tox envs are now defined:

- py27
- py27-coverage
- py36
- py36-coverage
---
 tox.ini | 42 ++++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/tox.ini b/tox.ini
index 98ca90c39..fe3e651b2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,13 +44,32 @@ usedevelop = False
 # We use extras=test to get things like "mock" that are required for our unit
 # tests.
 extras = test
-commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
-         tahoe --version
 
-[testenv:py36]
+setenv =
+       # Define TEST_SUITE in the environment as an aid to constructing the
+       # correct test command below.
+       !py36: TEST_SUITE = allmydata
+       py36: TEST_SUITE = allmydata.test.python3_tests
+
 commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
+	 # As an aid to debugging, dump all of the Python packages and their
+	 # versions that are installed in the test environment.  This is
+	 # particularly useful to get from CI runs - though hopefully the
+	 # version pinning we do limits the variability of this output
+         pip freeze
+
+	 # The tahoe script isn't sufficiently ported for this to succeed on
+	 # Python 3.x yet.
+         !py36: tahoe --version
+
+         !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
+
+	 # measuring coverage is somewhat slower than not measuring coverage
+	 # so only do it on request.
+	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+	 coverage: coverage combine
+         coverage: coverage xml
+
 
 [testenv:integration]
 setenv =
@@ -61,19 +80,6 @@ commands =
          coverage combine
          coverage report
 
-[testenv:coverage]
-# coverage (with --branch) takes about 65% longer to run
-commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
-	 # somewhat.
-         pip freeze
-         tahoe --version
-         coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
-         coverage combine
-         coverage xml
 
 [testenv:codechecks]
 # On macOS, git inside of towncrier needs $HOME.

From 07e33e78a5dcf55aa23cbde6f7edf58d56d120d1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:37:21 -0400
Subject: [PATCH 33/59] Configure CI to use the py36-coverage tox env

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c15eb1746..df181f058 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -285,7 +285,7 @@ jobs:
       # this reporter on Python 3.  So drop that and just specify the
       # reporter.
       TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
 
 
   ubuntu-20.04:

From 7f3192e310fdee14675d9a2b1ce661e1b3b9824f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 26 Aug 2020 14:46:57 -0400
Subject: [PATCH 34/59] news fragment

---
 newsfragments/3355.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3355.minor

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
new file mode 100644
index 000000000..e69de29bb

From e107e110792033b90f8e13eed1b7caba890d6bee Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:37:37 -0400
Subject: [PATCH 35/59] Remove references to the bare "coverage" tox
 environment

* We stopped using Appveyor a while ago so entirely remove its configuration.
* There's no release step where coverage information is *examined* so why
  collect it?
* Switch GitHub Actions config to py27-coverage tox environment
---
 .appveyor.yml                             | 95 -----------------------
 .github/workflows/ci.yml                  |  4 +-
 docs/how_to_make_a_tahoe-lafs_release.org |  2 +-
 3 files changed, 3 insertions(+), 98 deletions(-)
 delete mode 100644 .appveyor.yml

diff --git a/.appveyor.yml b/.appveyor.yml
deleted file mode 100644
index f6efe785a..000000000
--- a/.appveyor.yml
+++ /dev/null
@@ -1,95 +0,0 @@
-# adapted from https://packaging.python.org/en/latest/appveyor/
-
-environment:
-
-  matrix:
-
-    # For Python versions available on Appveyor, see
-    # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
-    #  DISTUTILS_USE_SDK: "1"
-    #  TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
-
-install:
-  - |
-    %PYTHON%\python.exe -m pip install -U pip
-    %PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
-
-# note:
-# %PYTHON% has: python.exe
-# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
-
-# We have a custom "build" system.  We don't need MSBuild or whatever.
-build: off
-
-# Do not build feature branch with open pull requests.  This is documented but
-# it's not clear it does anything.
-skip_branch_with_pr: true
-
-# This, perhaps, is effective.
-branches:
-  # whitelist
-  only:
-    - 'master'
-
-skip_commits:
- files:
-   # The Windows builds are unaffected by news fragments.
-   - 'newsfragments/*'
-   # Also, all this build junk.
-   - '.circleci/*'
-   - '.lgtm.yml'
-   - '.travis.yml'
-
-# we run from C:\projects\tahoe-lafs
-
-test_script:
-  # Put your test command here.
-  # Note that you must use the environment variable %PYTHON% to refer to
-  # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python version you want to use on PATH.
-  - |
-    %PYTHON%\Scripts\tox.exe -e coverage
-    %PYTHON%\Scripts\tox.exe -e pyinstaller
-  # To verify that the resultant PyInstaller-generated binary executes
-  # cleanly (i.e., that it terminates with an exit code of 0 and isn't
-  # failing due to import/packaging-related errors, etc.).
-  - dist\Tahoe-LAFS\tahoe.exe --version
-
-after_test:
-  # This builds the main tahoe wheel, and wheels for all dependencies.
-  # Again, you only need build.cmd if you're building C extensions for
-  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
-  # interpreter. If _trial_temp still exists, the "pip wheel" fails on
-  # _trial_temp\local_dir (not sure why).
-  - |
-    copy _trial_temp\test.log trial_test_log.txt
-    rd /s /q _trial_temp
-    %PYTHON%\python.exe setup.py bdist_wheel
-    %PYTHON%\python.exe -m pip wheel -w dist .
-  - |
-    %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
-    %PYTHON%\python.exe -m coverage xml -o coverage.xml -i
-    %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-  # "pip wheel -w dist ." puts all the dependency wheels there too
-  # this gives us a zipfile with everything
-  - path: 'dist\*'
-  - path: trial_test_log.txt
-    name: Trial test.log
-  - path: eliot.log
-    name: Eliot test log
-
-on_failure:
-  # Artifacts are not normally uploaded when the job fails.  To get the test
-  # logs, we have to push them ourselves.
-  - ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
-  - ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7cd97dcca..34a4e0875 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,8 +49,8 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e coverage"
-        run: tox -e coverage
+      - name: Run "tox -e py27-coverage"
+        run: tox -e py27-coverage
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1
diff --git a/docs/how_to_make_a_tahoe-lafs_release.org b/docs/how_to_make_a_tahoe-lafs_release.org
index 44b9e3dd1..b3f2a84d7 100644
--- a/docs/how_to_make_a_tahoe-lafs_release.org
+++ b/docs/how_to_make_a_tahoe-lafs_release.org
@@ -36,7 +36,7 @@ people are Release Maintainers:
   - [ ] documentation is ready (see above)
   - [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
   - [ ] build code locally:
-     tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
+     tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
   - [ ] created tarballs (they'll be in dist/ for later comparison)
      tox -e tarballs
   - [ ] release version is reporting itself as intended version

From 6422dba90de0bfd48c41c2a7b68ccd5726a03b36 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:39:49 -0400
Subject: [PATCH 36/59] Say some words about this change

---
 newsfragments/3355.minor | 0
 newsfragments/3355.other | 1 +
 2 files changed, 1 insertion(+)
 delete mode 100644 newsfragments/3355.minor
 create mode 100644 newsfragments/3355.other

diff --git a/newsfragments/3355.minor b/newsfragments/3355.minor
deleted file mode 100644
index e69de29bb..000000000
diff --git a/newsfragments/3355.other b/newsfragments/3355.other
new file mode 100644
index 000000000..4e854e4dd
--- /dev/null
+++ b/newsfragments/3355.other
@@ -0,0 +1 @@
+The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

From 0ed4f81e62bbb223b6d71beec11b32d8d1cd5476 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Aug 2020 10:40:27 -0400
Subject: [PATCH 37/59] Replace tabs with spaces

---
 tox.ini | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/tox.ini b/tox.ini
index fe3e651b2..2cb59388f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -52,22 +52,22 @@ setenv =
        py36: TEST_SUITE = allmydata.test.python3_tests
 
 commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
+         # As an aid to debugging, dump all of the Python packages and their
+         # versions that are installed in the test environment.  This is
+         # particularly useful to get from CI runs - though hopefully the
+         # version pinning we do limits the variability of this output
          pip freeze
 
-	 # The tahoe script isn't sufficiently ported for this to succeed on
-	 # Python 3.x yet.
+         # The tahoe script isn't sufficiently ported for this to succeed on
+         # Python 3.x yet.
          !py36: tahoe --version
 
          !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
 
-	 # measuring coverage is somewhat slower than not measuring coverage
-	 # so only do it on request.
-	 coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
-	 coverage: coverage combine
+         # measuring coverage is somewhat slower than not measuring coverage
+         # so only do it on request.
+         coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+         coverage: coverage combine
          coverage: coverage xml
 
 
@@ -75,8 +75,8 @@ commands =
 setenv =
          COVERAGE_PROCESS_START=.coveragerc
 commands =
-	 # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
-	 py.test --coverage -v {posargs:integration}
+         # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures'
+         py.test --coverage -v {posargs:integration}
          coverage combine
          coverage report
 
@@ -93,11 +93,11 @@ commands =
          python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
          python misc/coding_tools/check-miscaptures.py
 
-	 # If towncrier.check fails, you forgot to add a towncrier news
-	 # fragment explaining the change in this branch.  Create one at
-	 # `newsfragments/<ticket>.<change type>` with some text for the news
-	 # file.  See pyproject.toml for legal <change type> values.
-	 python -m towncrier.check --pyproject towncrier.pyproject.toml
+         # If towncrier.check fails, you forgot to add a towncrier news
+         # fragment explaining the change in this branch.  Create one at
+         # `newsfragments/<ticket>.<change type>` with some text for the news
+         # file.  See pyproject.toml for legal <change type> values.
+         python -m towncrier.check --pyproject towncrier.pyproject.toml
 
 [testenv:draftnews]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
@@ -116,9 +116,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7
@@ -144,9 +144,9 @@ commands =
          #
          # Some discussion is available at
          # https://github.com/pypa/pip/issues/5696
-	 #
-	 # towncrier post 19.2 (unreleased as of this writing) adds a --config
-	 # option that can be used instead of this file shuffling.
+         #
+         # towncrier post 19.2 (unreleased as of this writing) adds a --config
+         # option that can be used instead of this file shuffling.
          mv towncrier.pyproject.toml pyproject.toml
 
          # towncrier 19.2 + works with python2.7

From c3494f1356fa0d648fcbe139d9243f330fdb566c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:19:49 -0400
Subject: [PATCH 38/59] Enough changes to make allmydata.test.test_storage run
 on Python 3.

Still lots of failures, of course.
---
 src/allmydata/dirnode.py                           | 4 +++-
 src/allmydata/immutable/downloader/fetcher.py      | 2 +-
 src/allmydata/immutable/downloader/finder.py       | 2 +-
 src/allmydata/immutable/downloader/node.py         | 8 ++++----
 src/allmydata/immutable/downloader/segmentation.py | 2 +-
 src/allmydata/immutable/downloader/share.py        | 2 +-
 src/allmydata/node.py                              | 9 +++++++--
 src/allmydata/test/eliotutil.py                    | 2 ++
 8 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py
index 38cb26caf..59ebd73ba 100644
--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@@ -1,4 +1,6 @@
 """Directory Node implementation."""
+from past.builtins import unicode
+
 import time
 
 from zope.interface import implementer
@@ -227,7 +229,7 @@ def pack_children(childrenx, writekey, deep_immutable=False):
     return _pack_normalized_children(children, writekey=writekey, deep_immutable=deep_immutable)
 
 
-ZERO_LEN_NETSTR=netstring('')
+ZERO_LEN_NETSTR=netstring(b'')
 def _pack_normalized_children(children, writekey, deep_immutable=False):
     """Take a dict that maps:
          children[unicode_nfc_name] = (IFileSystemNode, metadata_dict)
diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py
index f3cd41fe0..a747fda6c 100644
--- a/src/allmydata/immutable/downloader/fetcher.py
+++ b/src/allmydata/immutable/downloader/fetcher.py
@@ -4,7 +4,7 @@ from foolscap.api import eventually
 from allmydata.interfaces import NotEnoughSharesError, NoSharesError
 from allmydata.util import log
 from allmydata.util.dictutil import DictOfSets
-from common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
+from .common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
      BadSegmentNumberError
 
 class SegmentFetcher(object):
diff --git a/src/allmydata/immutable/downloader/finder.py b/src/allmydata/immutable/downloader/finder.py
index a60aee814..43d5b78a4 100644
--- a/src/allmydata/immutable/downloader/finder.py
+++ b/src/allmydata/immutable/downloader/finder.py
@@ -5,7 +5,7 @@ from foolscap.api import eventually
 from allmydata.util import base32, log
 from twisted.internet import reactor
 
-from share import Share, CommonShare
+from .share import Share, CommonShare
 
 def incidentally(res, f, *args, **kwargs):
     """Add me to a Deferred chain like this:
diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py
index 50a8a2ce3..f67278132 100644
--- a/src/allmydata/immutable/downloader/node.py
+++ b/src/allmydata/immutable/downloader/node.py
@@ -13,10 +13,10 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
      NotEnoughHashesError
 
 # local imports
-from finder import ShareFinder
-from fetcher import SegmentFetcher
-from segmentation import Segmentation
-from common import BadCiphertextHashError
+from .finder import ShareFinder
+from .fetcher import SegmentFetcher
+from .segmentation import Segmentation
+from .common import BadCiphertextHashError
 
 class IDownloadStatusHandlingConsumer(Interface):
     def set_download_status_read_event(read_ev):
diff --git a/src/allmydata/immutable/downloader/segmentation.py b/src/allmydata/immutable/downloader/segmentation.py
index ea019d76a..6f634da0b 100644
--- a/src/allmydata/immutable/downloader/segmentation.py
+++ b/src/allmydata/immutable/downloader/segmentation.py
@@ -9,7 +9,7 @@ from allmydata.util import log
 from allmydata.util.spans import overlap
 from allmydata.interfaces import DownloadStopped
 
-from common import BadSegmentNumberError, WrongSegmentError
+from .common import BadSegmentNumberError, WrongSegmentError
 
 @implementer(IPushProducer)
 class Segmentation(object):
diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py
index 5237a7a9b..0da563baa 100644
--- a/src/allmydata/immutable/downloader/share.py
+++ b/src/allmydata/immutable/downloader/share.py
@@ -13,7 +13,7 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
 
 from allmydata.immutable.layout import make_write_bucket_proxy
 from allmydata.util.observer import EventStreamObserver
-from common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
+from .common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
 
 
 class LayoutInvalid(Exception):
diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 95033d36c..aad90393c 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -2,6 +2,8 @@
 This module contains classes and functions to implement and manage
 a node for Tahoe-LAFS.
 """
+from past.builtins import unicode
+
 import datetime
 import os.path
 import re
@@ -70,7 +72,7 @@ def _common_valid_config():
 
 # Add our application versions to the data that Foolscap's LogPublisher
 # reports.
-for thing, things_version in get_package_versions().iteritems():
+for thing, things_version in get_package_versions().items():
     app_versions.add_version(thing, str(things_version))
 
 # group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
@@ -275,7 +277,10 @@ class _Config(object):
         self.config = configparser
 
         nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
-        self.nickname = nickname_utf8.decode("utf-8")
+        if isinstance(nickname_utf8, bytes):  # Python 2
+            self.nickname = nickname_utf8.decode("utf-8")
+        else:
+            self.nickname = nickname_utf8
         assert type(self.nickname) is unicode
 
     def validate(self, valid_config_sections):
diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index f5972c929..7e107fdcb 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -2,6 +2,8 @@
 Tools aimed at the interaction between tests and Eliot.
 """
 
+from past.builtins import unicode
+
 __all__ = [
     "RUN_TEST",
     "EliotLoggedRunTest",

From 6007c1f67f3b6350ddf5c34966c236526d90092e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:36:54 -0400
Subject: [PATCH 39/59] Some tests are passing.

---
 src/allmydata/immutable/layout.py  | 10 +--
 src/allmydata/test/test_storage.py | 98 +++++++++++++++---------------
 2 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py
index 320f742e0..5d1f691b9 100644
--- a/src/allmydata/immutable/layout.py
+++ b/src/allmydata/immutable/layout.py
@@ -171,7 +171,7 @@ class WriteBucketProxy(object):
     def put_block(self, segmentnum, data):
         offset = self._offsets['data'] + segmentnum * self._block_size
         assert offset + len(data) <= self._offsets['uri_extension']
-        assert isinstance(data, str)
+        assert isinstance(data, bytes)
         if segmentnum < self._num_segments-1:
             precondition(len(data) == self._block_size,
                          len(data), self._block_size)
@@ -185,7 +185,7 @@ class WriteBucketProxy(object):
     def put_crypttext_hashes(self, hashes):
         offset = self._offsets['crypttext_hash_tree']
         assert isinstance(hashes, list)
-        data = "".join(hashes)
+        data = b"".join(hashes)
         precondition(len(data) == self._segment_hash_size,
                      len(data), self._segment_hash_size)
         precondition(offset + len(data) <= self._offsets['block_hashes'],
@@ -196,7 +196,7 @@ class WriteBucketProxy(object):
     def put_block_hashes(self, blockhashes):
         offset = self._offsets['block_hashes']
         assert isinstance(blockhashes, list)
-        data = "".join(blockhashes)
+        data = b"".join(blockhashes)
         precondition(len(data) == self._segment_hash_size,
                      len(data), self._segment_hash_size)
         precondition(offset + len(data) <= self._offsets['share_hashes'],
@@ -209,7 +209,7 @@ class WriteBucketProxy(object):
         # as 2+32=34 bytes each
         offset = self._offsets['share_hashes']
         assert isinstance(sharehashes, list)
-        data = "".join([struct.pack(">H", hashnum) + hashvalue
+        data = b"".join([struct.pack(">H", hashnum) + hashvalue
                         for hashnum,hashvalue in sharehashes])
         precondition(len(data) == self._share_hashtree_size,
                      len(data), self._share_hashtree_size)
@@ -220,7 +220,7 @@ class WriteBucketProxy(object):
 
     def put_uri_extension(self, data):
         offset = self._offsets['uri_extension']
-        assert isinstance(data, str)
+        assert isinstance(data, bytes)
         precondition(len(data) <= self._uri_extension_size_max,
                      len(data), self._uri_extension_size_max)
         length = struct.pack(self.fieldstruct, len(data))
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index d04e6b83d..9395ff8e7 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -65,32 +65,32 @@ class Bucket(unittest.TestCase):
         cancel_secret = os.urandom(32)
         expiration_time = time.time() + 5000
         return LeaseInfo(owner_num, renew_secret, cancel_secret,
-                         expiration_time, "\x00" * 20)
+                         expiration_time, b"\x00" * 20)
 
     def test_create(self):
         incoming, final = self.make_workdir("test_create")
         bw = BucketWriter(self, incoming, final, 200, self.make_lease(),
                           FakeCanary())
-        bw.remote_write(0, "a"*25)
-        bw.remote_write(25, "b"*25)
-        bw.remote_write(50, "c"*25)
-        bw.remote_write(75, "d"*7)
+        bw.remote_write(0, b"a"*25)
+        bw.remote_write(25, b"b"*25)
+        bw.remote_write(50, b"c"*25)
+        bw.remote_write(75, b"d"*7)
         bw.remote_close()
 
     def test_readwrite(self):
         incoming, final = self.make_workdir("test_readwrite")
         bw = BucketWriter(self, incoming, final, 200, self.make_lease(),
                           FakeCanary())
-        bw.remote_write(0, "a"*25)
-        bw.remote_write(25, "b"*25)
-        bw.remote_write(50, "c"*7) # last block may be short
+        bw.remote_write(0, b"a"*25)
+        bw.remote_write(25, b"b"*25)
+        bw.remote_write(50, b"c"*7) # last block may be short
         bw.remote_close()
 
         # now read from it
         br = BucketReader(self, bw.finalhome)
-        self.failUnlessEqual(br.remote_read(0, 25), "a"*25)
-        self.failUnlessEqual(br.remote_read(25, 25), "b"*25)
-        self.failUnlessEqual(br.remote_read(50, 7), "c"*7)
+        self.failUnlessEqual(br.remote_read(0, 25), b"a"*25)
+        self.failUnlessEqual(br.remote_read(25, 25), b"b"*25)
+        self.failUnlessEqual(br.remote_read(50, 7), b"c"*7)
 
     def test_read_past_end_of_share_data(self):
         # test vector for immutable files (hard-coded contents of an immutable share
@@ -107,12 +107,12 @@ class Bucket(unittest.TestCase):
         # complicated string involving hash trees and a URI Extension Block
         # -- see allmydata/immutable/layout.py . This test, which is
         # simulating a client, just sends 'a'.
-        share_data = 'a'
+        share_data = b'a'
 
         ownernumber = struct.pack('>L', 0)
-        renewsecret  = 'THIS LETS ME RENEW YOUR FILE....'
+        renewsecret  = b'THIS LETS ME RENEW YOUR FILE....'
         assert len(renewsecret) == 32
-        cancelsecret = 'THIS LETS ME KILL YOUR FILE HAHA'
+        cancelsecret = b'THIS LETS ME KILL YOUR FILE HAHA'
         assert len(cancelsecret) == 32
         expirationtime = struct.pack('>L', 60*60*24*31) # 31 days in seconds
 
@@ -184,7 +184,7 @@ class BucketProxy(unittest.TestCase):
         cancel_secret = os.urandom(32)
         expiration_time = time.time() + 5000
         return LeaseInfo(owner_num, renew_secret, cancel_secret,
-                         expiration_time, "\x00" * 20)
+                         expiration_time, b"\x00" * 20)
 
     def bucket_writer_closed(self, bw, consumed):
         pass
@@ -217,13 +217,13 @@ class BucketProxy(unittest.TestCase):
 
         sharesize = header_size + 100 + 7*32 + 7*32 + 7*32 + 3*(2+32) + 4+500
 
-        crypttext_hashes = [hashutil.tagged_hash("crypt", "bar%d" % i)
+        crypttext_hashes = [hashutil.tagged_hash(b"crypt", b"bar%d" % i)
                             for i in range(7)]
-        block_hashes = [hashutil.tagged_hash("block", "bar%d" % i)
+        block_hashes = [hashutil.tagged_hash(b"block", b"bar%d" % i)
                         for i in range(7)]
-        share_hashes = [(i, hashutil.tagged_hash("share", "bar%d" % i))
+        share_hashes = [(i, hashutil.tagged_hash(b"share", b"bar%d" % i))
                         for i in (1,9,13)]
-        uri_extension = "s" + "E"*498 + "e"
+        uri_extension = b"s" + b"E"*498 + b"e"
 
         bw, rb, sharefname = self.make_bucket(name, sharesize)
         bp = wbp_class(rb, None,
@@ -234,10 +234,10 @@ class BucketProxy(unittest.TestCase):
                        uri_extension_size_max=len(uri_extension))
 
         d = bp.put_header()
-        d.addCallback(lambda res: bp.put_block(0, "a"*25))
-        d.addCallback(lambda res: bp.put_block(1, "b"*25))
-        d.addCallback(lambda res: bp.put_block(2, "c"*25))
-        d.addCallback(lambda res: bp.put_block(3, "d"*20))
+        d.addCallback(lambda res: bp.put_block(0, b"a"*25))
+        d.addCallback(lambda res: bp.put_block(1, b"b"*25))
+        d.addCallback(lambda res: bp.put_block(2, b"c"*25))
+        d.addCallback(lambda res: bp.put_block(3, b"d"*20))
         d.addCallback(lambda res: bp.put_crypttext_hashes(crypttext_hashes))
         d.addCallback(lambda res: bp.put_block_hashes(block_hashes))
         d.addCallback(lambda res: bp.put_share_hashes(share_hashes))
@@ -248,19 +248,19 @@ class BucketProxy(unittest.TestCase):
         def _start_reading(res):
             br = BucketReader(self, sharefname)
             rb = RemoteBucket(br)
-            server = NoNetworkServer("abc", None)
-            rbp = rbp_class(rb, server, storage_index="")
+            server = NoNetworkServer(b"abc", None)
+            rbp = rbp_class(rb, server, storage_index=b"")
             self.failUnlessIn("to peer", repr(rbp))
             self.failUnless(interfaces.IStorageBucketReader.providedBy(rbp), rbp)
 
             d1 = rbp.get_block_data(0, 25, 25)
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "a"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"a"*25))
             d1.addCallback(lambda res: rbp.get_block_data(1, 25, 25))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "b"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"b"*25))
             d1.addCallback(lambda res: rbp.get_block_data(2, 25, 25))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "c"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"c"*25))
             d1.addCallback(lambda res: rbp.get_block_data(3, 25, 20))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "d"*20))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"d"*20))
 
             d1.addCallback(lambda res: rbp.get_crypttext_hashes())
             d1.addCallback(lambda res:
@@ -302,7 +302,7 @@ class Server(unittest.TestCase):
 
     def create(self, name, reserved_space=0, klass=StorageServer):
         workdir = self.workdir(name)
-        ss = klass(workdir, "\x00" * 20, reserved_space=reserved_space,
+        ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space,
                    stats_provider=FakeStatsProvider())
         ss.setServiceParent(self.sparent)
         return ss
@@ -330,8 +330,8 @@ class Server(unittest.TestCase):
         self.failUnlessIn('available-space', sv1)
 
     def allocate(self, ss, storage_index, sharenums, size, canary=None):
-        renew_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())
-        cancel_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())
+        renew_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))
+        cancel_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))
         if not canary:
             canary = FakeCanary()
         return ss.remote_allocate_buckets(storage_index,
@@ -390,7 +390,7 @@ class Server(unittest.TestCase):
 
     def test_remove_incoming(self):
         ss = self.create("test_remove_incoming")
-        already, writers = self.allocate(ss, "vid", range(3), 10)
+        already, writers = self.allocate(ss, b"vid", range(3), 10)
         for i,wb in writers.items():
             wb.remote_write(0, "%10d" % i)
             wb.remote_close()
@@ -524,14 +524,14 @@ class Server(unittest.TestCase):
         OVERHEAD = 3*4
         LEASE_SIZE = 4+32+32+4
         canary = FakeCanary(True)
-        already, writers = self.allocate(ss, "vid1", [0,1,2], 1000, canary)
+        already, writers = self.allocate(ss, b"vid1", [0,1,2], 1000, canary)
         self.failUnlessEqual(len(writers), 3)
         # now the StorageServer should have 3000 bytes provisionally
         # allocated, allowing only 2000 more to be claimed
         self.failUnlessEqual(len(ss._active_writers), 3)
 
         # allocating 1001-byte shares only leaves room for one
-        already2, writers2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary)
+        already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary)
         self.failUnlessEqual(len(writers2), 1)
         self.failUnlessEqual(len(ss._active_writers), 4)
 
@@ -578,19 +578,19 @@ class Server(unittest.TestCase):
         fileutil.make_dirs(basedir)
         filename = os.path.join(basedir, "testfile")
         f = open(filename, "wb")
-        f.write("start")
+        f.write(b"start")
         f.close()
         # mode="w" allows seeking-to-create-holes, but truncates pre-existing
         # files. mode="a" preserves previous contents but does not allow
         # seeking-to-create-holes. mode="r+" allows both.
         f = open(filename, "rb+")
         f.seek(100)
-        f.write("100")
+        f.write(b"100")
         f.close()
         filelen = os.stat(filename)[stat.ST_SIZE]
         self.failUnlessEqual(filelen, 100+3)
         f2 = open(filename, "rb")
-        self.failUnlessEqual(f2.read(5), "start")
+        self.failUnlessEqual(f2.read(5), b"start")
 
 
     def test_leases(self):
@@ -689,10 +689,10 @@ class Server(unittest.TestCase):
 
     def test_readonly(self):
         workdir = self.workdir("test_readonly")
-        ss = StorageServer(workdir, "\x00" * 20, readonly_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, readonly_storage=True)
         ss.setServiceParent(self.sparent)
 
-        already,writers = self.allocate(ss, "vid", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"vid", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(writers, {})
 
@@ -706,25 +706,25 @@ class Server(unittest.TestCase):
     def test_discard(self):
         # discard is really only used for other tests, but we test it anyways
         workdir = self.workdir("test_discard")
-        ss = StorageServer(workdir, "\x00" * 20, discard_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
-        already,writers = self.allocate(ss, "vid", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"vid", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
         for i,wb in writers.items():
-            wb.remote_write(0, "%25d" % i)
+            wb.remote_write(0, b"%25d" % i)
             wb.remote_close()
         # since we discard the data, the shares should be present but sparse.
         # Since we write with some seeks, the data we read back will be all
         # zeros.
-        b = ss.remote_get_buckets("vid")
+        b = ss.remote_get_buckets(b"vid")
         self.failUnlessEqual(set(b.keys()), set([0,1,2]))
-        self.failUnlessEqual(b[0].remote_read(0, 25), "\x00" * 25)
+        self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25)
 
     def test_advise_corruption(self):
         workdir = self.workdir("test_advise_corruption")
-        ss = StorageServer(workdir, "\x00" * 20, discard_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
         si0_s = base32.b2a("si0")
@@ -782,7 +782,7 @@ class MutableServer(unittest.TestCase):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 
@@ -1489,7 +1489,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 
@@ -2873,7 +2873,7 @@ class Stats(unittest.TestCase):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 

From 9ce43231b492a0e8f82f55b1a4d1b58bccf9067d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:49:04 -0400
Subject: [PATCH 40/59] More passing tests.

---
 src/allmydata/storage/server.py    |   2 +-
 src/allmydata/test/test_storage.py | 164 +++++++++++++++--------------
 2 files changed, 84 insertions(+), 82 deletions(-)

diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 3ffb58b68..387225b01 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -398,7 +398,7 @@ class StorageServer(service.MultiService, Referenceable):
         # since all shares get the same lease data, we just grab the leases
         # from the first share
         try:
-            shnum, filename = self._get_bucket_shares(storage_index).next()
+            shnum, filename = next(self._get_bucket_shares(storage_index))
             sf = ShareFile(filename)
             return sf.get_leases()
         except StopIteration:
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 9395ff8e7..bf0dda179 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,3 +1,5 @@
+from future.utils import native_str
+
 import time
 import os.path
 import platform
@@ -349,18 +351,18 @@ class Server(unittest.TestCase):
 
         ss = self.create("test_large_share")
 
-        already,writers = self.allocate(ss, "allocate", [0], 2**32+2)
+        already,writers = self.allocate(ss, b"allocate", [0], 2**32+2)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0]))
 
-        shnum, bucket = writers.items()[0]
+        shnum, bucket = list(writers.items())[0]
         # This test is going to hammer your filesystem if it doesn't make a sparse file for this.  :-(
-        bucket.remote_write(2**32, "ab")
+        bucket.remote_write(2**32, b"ab")
         bucket.remote_close()
 
-        readers = ss.remote_get_buckets("allocate")
+        readers = ss.remote_get_buckets(b"allocate")
         reader = readers[shnum]
-        self.failUnlessEqual(reader.remote_read(2**32, 2), "ab")
+        self.failUnlessEqual(reader.remote_read(2**32, 2), b"ab")
 
     def test_dont_overfill_dirs(self):
         """
@@ -369,9 +371,9 @@ class Server(unittest.TestCase):
         same storage index), this won't add an entry to the share directory.
         """
         ss = self.create("test_dont_overfill_dirs")
-        already, writers = self.allocate(ss, "storageindex", [0], 10)
+        already, writers = self.allocate(ss, b"storageindex", [0], 10)
         for i, wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         storedir = os.path.join(self.workdir("test_dont_overfill_dirs"),
                                 "shares")
@@ -379,9 +381,9 @@ class Server(unittest.TestCase):
 
         # Now store another one under another storageindex that has leading
         # chars the same as the first storageindex.
-        already, writers = self.allocate(ss, "storageindey", [0], 10)
+        already, writers = self.allocate(ss, b"storageindey", [0], 10)
         for i, wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         storedir = os.path.join(self.workdir("test_dont_overfill_dirs"),
                                 "shares")
@@ -392,7 +394,7 @@ class Server(unittest.TestCase):
         ss = self.create("test_remove_incoming")
         already, writers = self.allocate(ss, b"vid", range(3), 10)
         for i,wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         incoming_share_dir = wb.incominghome
         incoming_bucket_dir = os.path.dirname(incoming_share_dir)
@@ -407,11 +409,11 @@ class Server(unittest.TestCase):
         # the allocated size of the bucket is not counted by the storage
         # server when accounting for space.
         ss = self.create("test_abort")
-        already, writers = self.allocate(ss, "allocate", [0, 1, 2], 150)
+        already, writers = self.allocate(ss, b"allocate", [0, 1, 2], 150)
         self.failIfEqual(ss.allocated_size(), 0)
 
         # Now abort the writers.
-        for writer in writers.itervalues():
+        for writer in writers.values():
             writer.remote_abort()
         self.failUnlessEqual(ss.allocated_size(), 0)
 
@@ -419,26 +421,26 @@ class Server(unittest.TestCase):
     def test_allocate(self):
         ss = self.create("test_allocate")
 
-        self.failUnlessEqual(ss.remote_get_buckets("allocate"), {})
+        self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {})
 
-        already,writers = self.allocate(ss, "allocate", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"allocate", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
 
         # while the buckets are open, they should not count as readable
-        self.failUnlessEqual(ss.remote_get_buckets("allocate"), {})
+        self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {})
 
         # close the buckets
         for i,wb in writers.items():
-            wb.remote_write(0, "%25d" % i)
+            wb.remote_write(0, b"%25d" % i)
             wb.remote_close()
             # aborting a bucket that was already closed is a no-op
             wb.remote_abort()
 
         # now they should be readable
-        b = ss.remote_get_buckets("allocate")
+        b = ss.remote_get_buckets(b"allocate")
         self.failUnlessEqual(set(b.keys()), set([0,1,2]))
-        self.failUnlessEqual(b[0].remote_read(0, 25), "%25d" % 0)
+        self.failUnlessEqual(b[0].remote_read(0, 25), b"%25d" % 0)
         b_str = str(b[0])
         self.failUnlessIn("BucketReader", b_str)
         self.failUnlessIn("mfwgy33dmf2g 0", b_str)
@@ -446,21 +448,21 @@ class Server(unittest.TestCase):
         # now if we ask about writing again, the server should offer those
         # three buckets as already present. It should offer them even if we
         # don't ask about those specific ones.
-        already,writers = self.allocate(ss, "allocate", [2,3,4], 75)
+        already,writers = self.allocate(ss, b"allocate", [2,3,4], 75)
         self.failUnlessEqual(already, set([0,1,2]))
         self.failUnlessEqual(set(writers.keys()), set([3,4]))
 
         # while those two buckets are open for writing, the server should
         # refuse to offer them to uploaders
 
-        already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75)
+        already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75)
         self.failUnlessEqual(already2, set([0,1,2]))
         self.failUnlessEqual(set(writers2.keys()), set([5]))
 
         # aborting the writes should remove the tempfiles
         for i,wb in writers2.items():
             wb.remote_abort()
-        already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75)
+        already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75)
         self.failUnlessEqual(already2, set([0,1,2]))
         self.failUnlessEqual(set(writers2.keys()), set([5]))
 
@@ -471,27 +473,27 @@ class Server(unittest.TestCase):
 
     def test_bad_container_version(self):
         ss = self.create("test_bad_container_version")
-        a,w = self.allocate(ss, "si1", [0], 10)
-        w[0].remote_write(0, "\xff"*10)
+        a,w = self.allocate(ss, b"si1", [0], 10)
+        w[0].remote_write(0, b"\xff"*10)
         w[0].remote_close()
 
-        fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
+        fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
         f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1
         f.close()
 
-        ss.remote_get_buckets("allocate")
+        ss.remote_get_buckets(b"allocate")
 
         e = self.failUnlessRaises(UnknownImmutableContainerVersionError,
-                                  ss.remote_get_buckets, "si1")
+                                  ss.remote_get_buckets, b"si1")
         self.failUnlessIn(" had version 0 but we wanted 1", str(e))
 
     def test_disconnect(self):
         # simulate a disconnection
         ss = self.create("test_disconnect")
         canary = FakeCanary()
-        already,writers = self.allocate(ss, "disconnect", [0,1,2], 75, canary)
+        already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75, canary)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
         for (f,args,kwargs) in canary.disconnectors.values():
@@ -500,7 +502,7 @@ class Server(unittest.TestCase):
         del writers
 
         # that ought to delete the incoming shares
-        already,writers = self.allocate(ss, "disconnect", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
 
@@ -549,7 +551,7 @@ class Server(unittest.TestCase):
         # become real, long-term allocation, and grows to include the
         # overhead.
         for bw in writers2.values():
-            bw.remote_write(0, "a"*25)
+            bw.remote_write(0, b"a"*25)
             bw.remote_close()
         del already2
         del writers2
@@ -561,7 +563,7 @@ class Server(unittest.TestCase):
 
         # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
         # 5000-1085=3915 free, therefore we can fit 39 100byte shares
-        already3, writers3 = self.allocate(ss,"vid3", range(100), 100, canary)
+        already3, writers3 = self.allocate(ss, b"vid3", range(100), 100, canary)
         self.failUnlessEqual(len(writers3), 39)
         self.failUnlessEqual(len(ss._active_writers), 39)
 
@@ -599,60 +601,60 @@ class Server(unittest.TestCase):
         sharenums = range(5)
         size = 100
 
-        rs0,cs0 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si0", rs0, cs0,
+        rs0,cs0 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si0", rs0, cs0,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 0)
         self.failUnlessEqual(len(writers), 5)
         for wb in writers.values():
             wb.remote_close()
 
-        leases = list(ss.get_leases("si0"))
+        leases = list(ss.get_leases(b"si0"))
         self.failUnlessEqual(len(leases), 1)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0]))
 
-        rs1,cs1 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si1", rs1, cs1,
+        rs1,cs1 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si1", rs1, cs1,
                                                      sharenums, size, canary)
         for wb in writers.values():
             wb.remote_close()
 
         # take out a second lease on si1
-        rs2,cs2 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si1", rs2, cs2,
+        rs2,cs2 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si1", rs2, cs2,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 5)
         self.failUnlessEqual(len(writers), 0)
 
-        leases = list(ss.get_leases("si1"))
+        leases = list(ss.get_leases(b"si1"))
         self.failUnlessEqual(len(leases), 2)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2]))
 
         # and a third lease, using add-lease
-        rs2a,cs2a = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                     hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        ss.remote_add_lease("si1", rs2a, cs2a)
-        leases = list(ss.get_leases("si1"))
+        rs2a,cs2a = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                     hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        ss.remote_add_lease(b"si1", rs2a, cs2a)
+        leases = list(ss.get_leases(b"si1"))
         self.failUnlessEqual(len(leases), 3)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2, rs2a]))
 
         # add-lease on a missing storage index is silently ignored
-        self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None)
+        self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None)
 
         # check that si0 is readable
-        readers = ss.remote_get_buckets("si0")
+        readers = ss.remote_get_buckets(b"si0")
         self.failUnlessEqual(len(readers), 5)
 
         # renew the first lease. Only the proper renew_secret should work
-        ss.remote_renew_lease("si0", rs0)
-        self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", cs0)
-        self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", rs1)
+        ss.remote_renew_lease(b"si0", rs0)
+        self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", cs0)
+        self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", rs1)
 
         # check that si0 is still readable
-        readers = ss.remote_get_buckets("si0")
+        readers = ss.remote_get_buckets(b"si0")
         self.failUnlessEqual(len(readers), 5)
 
         # There is no such method as remote_cancel_lease for now -- see
@@ -661,30 +663,30 @@ class Server(unittest.TestCase):
                         "ss should not have a 'remote_cancel_lease' method/attribute")
 
         # test overlapping uploads
-        rs3,cs3 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        rs4,cs4 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si3", rs3, cs3,
+        rs3,cs3 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        rs4,cs4 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si3", rs3, cs3,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 0)
         self.failUnlessEqual(len(writers), 5)
-        already2,writers2 = ss.remote_allocate_buckets("si3", rs4, cs4,
+        already2,writers2 = ss.remote_allocate_buckets(b"si3", rs4, cs4,
                                                        sharenums, size, canary)
         self.failUnlessEqual(len(already2), 0)
         self.failUnlessEqual(len(writers2), 0)
         for wb in writers.values():
             wb.remote_close()
 
-        leases = list(ss.get_leases("si3"))
+        leases = list(ss.get_leases(b"si3"))
         self.failUnlessEqual(len(leases), 1)
 
-        already3,writers3 = ss.remote_allocate_buckets("si3", rs4, cs4,
+        already3,writers3 = ss.remote_allocate_buckets(b"si3", rs4, cs4,
                                                        sharenums, size, canary)
         self.failUnlessEqual(len(already3), 5)
         self.failUnlessEqual(len(writers3), 0)
 
-        leases = list(ss.get_leases("si3"))
+        leases = list(ss.get_leases(b"si3"))
         self.failUnlessEqual(len(leases), 2)
 
     def test_readonly(self):
@@ -727,44 +729,44 @@ class Server(unittest.TestCase):
         ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
-        si0_s = base32.b2a("si0")
-        ss.remote_advise_corrupt_share("immutable", "si0", 0,
+        si0_s = base32.b2a(b"si0")
+        ss.remote_advise_corrupt_share(b"immutable", b"si0", 0,
                                        "This share smells funny.\n")
         reportdir = os.path.join(workdir, "corruption-advisories")
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 1)
         report_si0 = reports[0]
-        self.failUnlessIn(si0_s, report_si0)
-        f = open(os.path.join(reportdir, report_si0), "r")
+        self.failUnlessIn(native_str(si0_s), report_si0)
+        f = open(os.path.join(reportdir, report_si0), "rb")
         report = f.read()
         f.close()
-        self.failUnlessIn("type: immutable", report)
-        self.failUnlessIn("storage_index: %s" % si0_s, report)
-        self.failUnlessIn("share_number: 0", report)
-        self.failUnlessIn("This share smells funny.", report)
+        self.failUnlessIn(b"type: immutable", report)
+        self.failUnlessIn(b"storage_index: %s" % si0_s, report)
+        self.failUnlessIn(b"share_number: 0", report)
+        self.failUnlessIn(b"This share smells funny.", report)
 
         # test the RIBucketWriter version too
-        si1_s = base32.b2a("si1")
-        already,writers = self.allocate(ss, "si1", [1], 75)
+        si1_s = base32.b2a(b"si1")
+        already,writers = self.allocate(ss, b"si1", [1], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([1]))
-        writers[1].remote_write(0, "data")
+        writers[1].remote_write(0, b"data")
         writers[1].remote_close()
 
-        b = ss.remote_get_buckets("si1")
+        b = ss.remote_get_buckets(b"si1")
         self.failUnlessEqual(set(b.keys()), set([1]))
         b[1].remote_advise_corrupt_share("This share tastes like dust.\n")
 
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 2)
         report_si1 = [r for r in reports if si1_s in r][0]
-        f = open(os.path.join(reportdir, report_si1), "r")
+        f = open(os.path.join(reportdir, report_si1), "rb")
         report = f.read()
         f.close()
-        self.failUnlessIn("type: immutable", report)
-        self.failUnlessIn("storage_index: %s" % si1_s, report)
-        self.failUnlessIn("share_number: 1", report)
-        self.failUnlessIn("This share tastes like dust.", report)
+        self.failUnlessIn(b"type: immutable", report)
+        self.failUnlessIn(b"storage_index: %s" % si1_s, report)
+        self.failUnlessIn(b"share_number: 1", report)
+        self.failUnlessIn(b"This share tastes like dust.", report)
 
 
 
@@ -817,7 +819,7 @@ class MutableServer(unittest.TestCase):
 
     def test_bad_magic(self):
         ss = self.create("test_bad_magic")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(), set([0]), 10)
+        self.allocate(ss, "si1", "we1", next(self._lease_secret), set([0]), 10)
         fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
@@ -831,7 +833,7 @@ class MutableServer(unittest.TestCase):
 
     def test_container_size(self):
         ss = self.create("test_container_size")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         read = ss.remote_slot_readv
         rstaraw = ss.remote_slot_testv_and_readv_and_writev
@@ -929,7 +931,7 @@ class MutableServer(unittest.TestCase):
 
     def test_allocate(self):
         ss = self.create("test_allocate")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
 
         read = ss.remote_slot_readv
@@ -1315,7 +1317,7 @@ class MutableServer(unittest.TestCase):
 
     def test_remove(self):
         ss = self.create("test_remove")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         readv = ss.remote_slot_readv
         writev = ss.remote_slot_testv_and_readv_and_writev

From 1a65dfa4abddaf758b43d428f2ce6f6a8d1d5e7b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:58:03 -0400
Subject: [PATCH 41/59] Some potential progress.

---
 src/allmydata/test/test_storage.py | 68 ++++++++++++++++--------------
 1 file changed, 37 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index bf0dda179..647050b13 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -792,13 +792,19 @@ class MutableServer(unittest.TestCase):
         self.create("test_create")
 
     def write_enabler(self, we_tag):
-        return hashutil.tagged_hash("we_blah", we_tag)
+        return hashutil.tagged_hash(b"we_blah", we_tag)
 
     def renew_secret(self, tag):
-        return hashutil.tagged_hash("renew_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % (tag,)
+        assert isinstance(tag, bytes)
+        return hashutil.tagged_hash(b"renew_blah", tag)
 
     def cancel_secret(self, tag):
-        return hashutil.tagged_hash("cancel_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % (tag,)
+        assert isinstance(tag, bytes)
+        return hashutil.tagged_hash(b"cancel_blah", tag)
 
     def allocate(self, ss, storage_index, we_tag, lease_tag, sharenums, size):
         write_enabler = self.write_enabler(we_tag)
@@ -819,29 +825,29 @@ class MutableServer(unittest.TestCase):
 
     def test_bad_magic(self):
         ss = self.create("test_bad_magic")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret), set([0]), 10)
-        fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10)
+        fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
-        f.write("BAD MAGIC")
+        f.write(b"BAD MAGIC")
         f.close()
         read = ss.remote_slot_readv
         e = self.failUnlessRaises(UnknownMutableContainerVersionError,
-                                  read, "si1", [0], [(0,10)])
+                                  read, b"si1", [0], [(0,10)])
         self.failUnlessIn(" had magic ", str(e))
         self.failUnlessIn(" but we wanted ", str(e))
 
     def test_container_size(self):
         ss = self.create("test_container_size")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         read = ss.remote_slot_readv
         rstaraw = ss.remote_slot_testv_and_readv_and_writev
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
-        answer = rstaraw("si1", secrets,
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], len(data)+12)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
@@ -850,33 +856,33 @@ class MutableServer(unittest.TestCase):
         # whose offset is too high) will raise an exception.
         TOOBIG = MutableShareFile.MAX_SIZE + 10
         self.failUnlessRaises(DataTooLargeError,
-                              rstaraw, "si1", secrets,
+                              rstaraw, b"si1", secrets,
                               {0: ([], [(TOOBIG,data)], None)},
                               [])
 
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], None)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {0: [data[:10]]})
 
         # Sending a new_length shorter than the current length truncates the
         # data.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 9)},
                          [])
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {0: [data[:9]]})
 
         # Sending a new_length longer than the current length doesn't change
         # the data.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 20)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
-        read_answer = read("si1", [0], [(0, 20)])
+        read_answer = read(b"si1", [0], [(0, 20)])
         self.failUnlessEqual(read_answer, {0: [data[:9]]})
 
         # Sending a write vector whose start is after the end of the current
@@ -884,35 +890,35 @@ class MutableServer(unittest.TestCase):
         # but instead fills with zeroes.
 
         # To test this, we fill the data area with a recognizable pattern.
-        pattern = ''.join([chr(i) for i in range(100)])
-        answer = rstaraw("si1", secrets,
+        pattern = u''.join([chr(i) for i in range(100)]).encode("utf-8")
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0, pattern)], None)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Then truncate the data...
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 20)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Just confirm that you get an empty string if you try to read from
         # past the (new) endpoint now.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], None)},
                          [(20, 1980)])
-        self.failUnlessEqual(answer, (True, {0:[''],1:[''],2:['']}))
+        self.failUnlessEqual(answer, (True, {0:[b''],1:[b''],2:[b'']}))
 
         # Then the extend the file by writing a vector which starts out past
         # the end...
-        answer = rstaraw("si1", secrets,
-                         {0: ([], [(50, 'hellothere')], None)},
+        answer = rstaraw(b"si1", secrets,
+                         {0: ([], [(50, b'hellothere')], None)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Now if you read the stuff between 20 (where we earlier truncated)
         # and 50, it had better be all zeroes.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], None)},
                          [(20, 30)])
-        self.failUnlessEqual(answer, (True, {0:['\x00'*30],1:[''],2:['']}))
+        self.failUnlessEqual(answer, (True, {0:[b'\x00'*30],1:[b''],2:[b'']}))
 
         # Also see if the server explicitly declares that it supports this
         # feature.
@@ -921,12 +927,12 @@ class MutableServer(unittest.TestCase):
         self.failUnless(storage_v1_ver.get("fills-holes-with-zero-bytes"))
 
         # If the size is dropped to zero the share is deleted.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], 0)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {})
 
     def test_allocate(self):

From 5ad5b79cdd1e7834ce7de0a2d5f399c8e0282dd6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 10:53:52 -0400
Subject: [PATCH 42/59] More passing tests.

---
 src/allmydata/storage/immutable.py |   2 +-
 src/allmydata/storage/mutable.py   |  18 +-
 src/allmydata/test/test_storage.py | 352 +++++++++++++++--------------
 3 files changed, 187 insertions(+), 185 deletions(-)

diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py
index 4ec70e1e7..32ba1a739 100644
--- a/src/allmydata/storage/immutable.py
+++ b/src/allmydata/storage/immutable.py
@@ -85,7 +85,7 @@ class ShareFile(object):
         seekpos = self._data_offset+offset
         actuallength = max(0, min(length, self._lease_offset-seekpos))
         if actuallength == 0:
-            return ""
+            return b""
         with open(self.home, 'rb') as f:
             f.seek(seekpos)
             return f.read(actuallength)
diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py
index c108dfe32..a1eddf6cc 100644
--- a/src/allmydata/storage/mutable.py
+++ b/src/allmydata/storage/mutable.py
@@ -113,7 +113,7 @@ class MutableShareFile(object):
             # start beyond the end of the data return an empty string.
             length = max(0, data_length-offset)
         if length == 0:
-            return ""
+            return b""
         precondition(offset+length <= data_length)
         f.seek(self.DATA_OFFSET+offset)
         data = f.read(length)
@@ -421,18 +421,18 @@ class MutableShareFile(object):
                     # self._change_container_size() here.
 
 def testv_compare(a, op, b):
-    assert op in ("lt", "le", "eq", "ne", "ge", "gt")
-    if op == "lt":
+    assert op in (b"lt", b"le", b"eq", b"ne", b"ge", b"gt")
+    if op == b"lt":
         return a < b
-    if op == "le":
+    if op == b"le":
         return a <= b
-    if op == "eq":
+    if op == b"eq":
         return a == b
-    if op == "ne":
+    if op == b"ne":
         return a != b
-    if op == "ge":
+    if op == b"ge":
         return a >= b
-    if op == "gt":
+    if op == b"gt":
         return a > b
     # never reached
 
@@ -441,7 +441,7 @@ class EmptyShare(object):
     def check_testv(self, testv):
         test_good = True
         for (offset, length, operator, specimen) in testv:
-            data = ""
+            data = b""
             if not testv_compare(data, operator, specimen):
                 test_good = False
                 break
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 647050b13..639780846 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,4 @@
-from future.utils import native_str
+from future.utils import native_str, PY3
 
 import time
 import os.path
@@ -937,72 +937,72 @@ class MutableServer(unittest.TestCase):
 
     def test_allocate(self):
         ss = self.create("test_allocate")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
 
         read = ss.remote_slot_readv
-        self.failUnlessEqual(read("si1", [0], [(0, 10)]),
-                             {0: [""]})
-        self.failUnlessEqual(read("si1", [], [(0, 10)]),
-                             {0: [""], 1: [""], 2: [""]})
-        self.failUnlessEqual(read("si1", [0], [(100, 10)]),
-                             {0: [""]})
+        self.failUnlessEqual(read(b"si1", [0], [(0, 10)]),
+                             {0: [b""]})
+        self.failUnlessEqual(read(b"si1", [], [(0, 10)]),
+                             {0: [b""], 1: [b""], 2: [b""]})
+        self.failUnlessEqual(read(b"si1", [0], [(100, 10)]),
+                             {0: [b""]})
 
         # try writing to one
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
-        answer = write("si1", secrets,
+        answer = write(b"si1", secrets,
                        {0: ([], [(0,data)], None)},
                        [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        self.failUnlessEqual(read("si1", [0], [(0,20)]),
-                             {0: ["00000000001111111111"]})
-        self.failUnlessEqual(read("si1", [0], [(95,10)]),
-                             {0: ["99999"]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,20)]),
+                             {0: [b"00000000001111111111"]})
+        self.failUnlessEqual(read(b"si1", [0], [(95,10)]),
+                             {0: [b"99999"]})
         #self.failUnlessEqual(s0.remote_get_length(), 100)
 
-        bad_secrets = ("bad write enabler", secrets[1], secrets[2])
+        bad_secrets = (b"bad write enabler", secrets[1], secrets[2])
         f = self.failUnlessRaises(BadWriteEnablerError,
-                                  write, "si1", bad_secrets,
+                                  write, b"si1", bad_secrets,
                                   {}, [])
-        self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", f)
+        self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", str(f))
 
         # this testv should fail
-        answer = write("si1", secrets,
-                       {0: ([(0, 12, "eq", "444444444444"),
-                             (20, 5, "eq", "22222"),
+        answer = write(b"si1", secrets,
+                       {0: ([(0, 12, b"eq", b"444444444444"),
+                             (20, 5, b"eq", b"22222"),
                              ],
-                            [(0, "x"*100)],
+                            [(0, b"x"*100)],
                             None),
                         },
                        [(0,12), (20,5)],
                        )
         self.failUnlessEqual(answer, (False,
-                                      {0: ["000000000011", "22222"],
-                                       1: ["", ""],
-                                       2: ["", ""],
+                                      {0: [b"000000000011", b"22222"],
+                                       1: [b"", b""],
+                                       2: [b"", b""],
                                        }))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
 
         # as should this one
-        answer = write("si1", secrets,
-                       {0: ([(10, 5, "lt", "11111"),
+        answer = write(b"si1", secrets,
+                       {0: ([(10, 5, b"lt", b"11111"),
                              ],
-                            [(0, "x"*100)],
+                            [(0, b"x"*100)],
                             None),
                         },
                        [(10,5)],
                        )
         self.failUnlessEqual(answer, (False,
-                                      {0: ["11111"],
-                                       1: [""],
-                                       2: [""]},
+                                      {0: [b"11111"],
+                                       1: [b""],
+                                       2: [b""]},
                                       ))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
 
 
     def test_operators(self):
@@ -1010,201 +1010,201 @@ class MutableServer(unittest.TestCase):
         # test both fail+pass, reset data after each one.
         ss = self.create("test_operators")
 
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
 
         def reset():
-            write("si1", secrets,
+            write(b"si1", secrets,
                   {0: ([], [(0,data)], None)},
                   [])
 
         reset()
 
         #  lt
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11110"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
-        self.failUnlessEqual(read("si1", [], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  le
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11110"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  eq
-        answer = write("si1", secrets, {0: ([(10, 5, "eq", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "eq", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  ne
-        answer = write("si1", secrets, {0: ([(10, 5, "ne", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ne", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  ge
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11110"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
         #  gt
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11110"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
         # finally, test some operators against empty shares
-        answer = write("si1", secrets, {1: ([(10, 5, "eq", "11112"),
+        answer = write(b"si1", secrets, {1: ([(10, 5, b"eq", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
     def test_readv(self):
         ss = self.create("test_readv")
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
-        data = [("%d" % i) * 100 for i in range(3)]
-        rc = write("si1", secrets,
+        data = [(b"%d" % i) * 100 for i in range(3)]
+        rc = write(b"si1", secrets,
                    {0: ([], [(0,data[0])], None),
                     1: ([], [(0,data[1])], None),
                     2: ([], [(0,data[2])], None),
                     }, [])
         self.failUnlessEqual(rc, (True, {}))
 
-        answer = read("si1", [], [(0, 10)])
-        self.failUnlessEqual(answer, {0: ["0"*10],
-                                      1: ["1"*10],
-                                      2: ["2"*10]})
+        answer = read(b"si1", [], [(0, 10)])
+        self.failUnlessEqual(answer, {0: [b"0"*10],
+                                      1: [b"1"*10],
+                                      2: [b"2"*10]})
 
     def compare_leases_without_timestamps(self, leases_a, leases_b):
         self.failUnlessEqual(len(leases_a), len(leases_b))
@@ -1230,19 +1230,19 @@ class MutableServer(unittest.TestCase):
     def test_leases(self):
         ss = self.create("test_leases")
         def secrets(n):
-            return ( self.write_enabler("we1"),
-                     self.renew_secret("we1-%d" % n),
-                     self.cancel_secret("we1-%d" % n) )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+            return ( self.write_enabler(b"we1"),
+                     self.renew_secret(b"we1-%d" % n),
+                     self.cancel_secret(b"we1-%d" % n) )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
-        rc = write("si1", secrets(0), {0: ([], [(0,data)], None)}, [])
+        rc = write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(rc, (True, {}))
 
         # create a random non-numeric file in the bucket directory, to
         # exercise the code that's supposed to ignore those.
         bucket_dir = os.path.join(self.workdir("test_leases"),
-                                  "shares", storage_index_to_dir("si1"))
+                                  "shares", storage_index_to_dir(b"si1"))
         f = open(os.path.join(bucket_dir, "ignore_me.txt"), "w")
         f.write("you ought to be ignoring me\n")
         f.close()
@@ -1251,45 +1251,45 @@ class MutableServer(unittest.TestCase):
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # add-lease on a missing storage index is silently ignored
-        self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None)
+        self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None)
 
         # re-allocate the slots and use the same secrets, that should update
         # the lease
-        write("si1", secrets(0), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # renew it directly
-        ss.remote_renew_lease("si1", secrets(0)[1])
+        ss.remote_renew_lease(b"si1", secrets(0)[1])
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # now allocate them with a bunch of different secrets, to trigger the
         # extended lease code. Use add_lease for one of them.
-        write("si1", secrets(1), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(1), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(len(list(s0.get_leases())), 2)
         secrets2 = secrets(2)
-        ss.remote_add_lease("si1", secrets2[1], secrets2[2])
+        ss.remote_add_lease(b"si1", secrets2[1], secrets2[2])
         self.failUnlessEqual(len(list(s0.get_leases())), 3)
-        write("si1", secrets(3), {0: ([], [(0,data)], None)}, [])
-        write("si1", secrets(4), {0: ([], [(0,data)], None)}, [])
-        write("si1", secrets(5), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(3), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(4), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(5), {0: ([], [(0,data)], None)}, [])
 
         self.failUnlessEqual(len(list(s0.get_leases())), 6)
 
         all_leases = list(s0.get_leases())
         # and write enough data to expand the container, forcing the server
         # to move the leases
-        write("si1", secrets(0),
+        write(b"si1", secrets(0),
               {0: ([], [(0,data)], 200), },
               [])
 
         # read back the leases, make sure they're still intact.
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
-        ss.remote_renew_lease("si1", secrets(0)[1])
-        ss.remote_renew_lease("si1", secrets(1)[1])
-        ss.remote_renew_lease("si1", secrets(2)[1])
-        ss.remote_renew_lease("si1", secrets(3)[1])
-        ss.remote_renew_lease("si1", secrets(4)[1])
+        ss.remote_renew_lease(b"si1", secrets(0)[1])
+        ss.remote_renew_lease(b"si1", secrets(1)[1])
+        ss.remote_renew_lease(b"si1", secrets(2)[1])
+        ss.remote_renew_lease(b"si1", secrets(3)[1])
+        ss.remote_renew_lease(b"si1", secrets(4)[1])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
         # get a new copy of the leases, with the current timestamps. Reading
         # data and failing to renew/cancel leases should leave the timestamps
@@ -1300,7 +1300,7 @@ class MutableServer(unittest.TestCase):
         # examine the exception thus raised, make sure the old nodeid is
         # present, to provide for share migration
         e = self.failUnlessRaises(IndexError,
-                                  ss.remote_renew_lease, "si1",
+                                  ss.remote_renew_lease, b"si1",
                                   secrets(20)[1])
         e_s = str(e)
         self.failUnlessIn("Unable to renew non-existent lease", e_s)
@@ -1310,56 +1310,58 @@ class MutableServer(unittest.TestCase):
         self.compare_leases(all_leases, list(s0.get_leases()))
 
         # reading shares should not modify the timestamp
-        read("si1", [], [(0,200)])
+        read(b"si1", [], [(0,200)])
         self.compare_leases(all_leases, list(s0.get_leases()))
 
-        write("si1", secrets(0),
-              {0: ([], [(200, "make me bigger")], None)}, [])
+        write(b"si1", secrets(0),
+              {0: ([], [(200, b"make me bigger")], None)}, [])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
-        write("si1", secrets(0),
-              {0: ([], [(500, "make me really bigger")], None)}, [])
+        write(b"si1", secrets(0),
+              {0: ([], [(500, b"make me really bigger")], None)}, [])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
     def test_remove(self):
         ss = self.create("test_remove")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         readv = ss.remote_slot_readv
         writev = ss.remote_slot_testv_and_readv_and_writev
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
         # delete sh0 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {0: ([], [], 0)},
                         [])
         # the answer should mention all the shares that existed before the
         # write
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
         # but a new read should show only sh1 and sh2
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
-                             {1: [""], 2: [""]})
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
+                             {1: [b""], 2: [b""]})
 
         # delete sh1 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {1: ([], [], 0)},
                         [])
         self.failUnlessEqual(answer, (True, {1:[],2:[]}) )
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
-                             {2: [""]})
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
+                             {2: [b""]})
 
         # delete sh2 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {2: ([], [], 0)},
                         [])
         self.failUnlessEqual(answer, (True, {2:[]}) )
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
                              {})
         # and the bucket directory should now be gone
-        si = base32.b2a("si1")
+        si = base32.b2a(b"si1")
         # note: this is a detail of the storage server implementation, and
         # may change in the future
+        if PY3:
+            si = si.decode("utf-8")
         prefix = si[:2]
         prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix)
         bucketdir = os.path.join(prefixdir, si)
@@ -1373,7 +1375,7 @@ class MutableServer(unittest.TestCase):
         """
         ss = self.create("test_writev_without_renew_lease")
 
-        storage_index = "si2"
+        storage_index = b"si2"
         secrets = (
             self.write_enabler(storage_index),
             self.renew_secret(storage_index),
@@ -1400,7 +1402,7 @@ class MutableServer(unittest.TestCase):
         When ``get_slot_leases`` is called for a slot for which the server has no
         shares, it returns an empty iterable.
         """
-        ss = self.create(b"test_get_slot_leases_empty_slot")
+        ss = self.create("test_get_slot_leases_empty_slot")
         self.assertEqual(
             list(ss.get_slot_leases(b"si1")),
             [],
@@ -1413,7 +1415,7 @@ class MutableServer(unittest.TestCase):
         """
         ss = self.create("test_remove_non_present")
 
-        storage_index = "si1"
+        storage_index = b"si1"
         secrets = (
             self.write_enabler(storage_index),
             self.renew_secret(storage_index),
@@ -1590,7 +1592,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # and the verification key
         data += self.verification_key
         # Then we'll add in gibberish until we get to the right point.
-        nulls = "".join([" " for i in xrange(len(data), share_data_offset)])
+        nulls = b"".join([b" " for i in xrange(len(data), share_data_offset)])
         data += nulls
 
         # Then the share data
@@ -1614,7 +1616,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         data = self.build_test_mdmf_share(tail_segment, empty)
         # Finally, we write the whole thing to the storage server in one
         # pass.
-        testvs = [(0, 1, "eq", "")]
+        testvs = [(0, 1, b"eq", b"")]
         tws = {}
         tws[0] = (testvs, [(0, data)], None)
         readv = [(0, 1)]
@@ -1655,7 +1657,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                               sharedata_offset,
                               encprivkey_offset,
                               eof_offset)
-        final_share = "".join([prefix,
+        final_share = b"".join([prefix,
                            offsets,
                            self.verification_key,
                            self.signature,
@@ -1680,7 +1682,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # read them. This method writes one, which resembles but is not
         write = self.ss.remote_slot_testv_and_readv_and_writev
         share = self.build_test_sdmf_share(empty)
-        testvs = [(0, 1, "eq", "")]
+        testvs = [(0, 1, b"eq", b"")]
         tws = {}
         tws[0] = (testvs, [(0, share)], None)
         readv = []
@@ -1690,7 +1692,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # Check that every method equals what we expect it to.
         d = defer.succeed(None)
         def _check_block_and_salt(block_and_salt):
@@ -1762,19 +1764,19 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_different_tail_segment_size(self):
         self.write_test_share_to_server("si1", tail_segment=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_block_and_salt(5)
         def _check_tail_segment(results):
             block, salt = results
             self.failUnlessEqual(len(block), 1)
-            self.failUnlessEqual(block, "a")
+            self.failUnlessEqual(block, b"a")
         d.addCallback(_check_tail_segment)
         return d
 
 
     def test_get_block_with_invalid_segnum(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "test invalid segnum",
@@ -1785,7 +1787,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_encoding_parameters_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_encoding_parameters()
         def _check_encoding_parameters(args):
             (k, n, segment_size, datalen) = args
@@ -1799,7 +1801,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_seqnum_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_seqnum()
         d.addCallback(lambda seqnum:
             self.failUnlessEqual(seqnum, 0))
@@ -1808,7 +1810,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_root_hash_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_root_hash()
         d.addCallback(lambda root_hash:
             self.failUnlessEqual(root_hash, self.root_hash))
@@ -1817,7 +1819,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_checkstring_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_checkstring()
         d.addCallback(lambda checkstring:
             self.failUnlessEqual(checkstring, self.checkstring))
@@ -1996,11 +1998,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def serialize_blockhashes(self, blockhashes):
-        return "".join(blockhashes)
+        return b"".join(blockhashes)
 
 
     def serialize_sharehashes(self, sharehashes):
-        ret = "".join([struct.pack(">H32s", i, sharehashes[i])
+        ret = b"".join([struct.pack(">H32s", i, sharehashes[i])
                         for i in sorted(sharehashes.keys())])
         return ret
 

From 50007ac868cd199538056bf820201c11b3a9347d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 12:41:19 -0400
Subject: [PATCH 43/59] More passing tests.

---
 src/allmydata/mutable/layout.py    |  73 +++++-----
 src/allmydata/test/test_storage.py | 208 +++++++++++++++--------------
 2 files changed, 143 insertions(+), 138 deletions(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index 20f2df3aa..5f459921a 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -1,3 +1,4 @@
+from past.utils import old_div
 
 import struct
 from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError, \
@@ -180,11 +181,11 @@ def pack_offsets(verification_key_length, signature_length,
 def pack_share(prefix, verification_key, signature,
                share_hash_chain, block_hash_tree,
                share_data, encprivkey):
-    share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
-                                  for i in sorted(share_hash_chain.keys())])
+    share_hash_chain_s = b"".join([struct.pack(">H32s", i, share_hash_chain[i])
+                                   for i in sorted(share_hash_chain.keys())])
     for h in block_hash_tree:
         assert len(h) == 32
-    block_hash_tree_s = "".join(block_hash_tree)
+    block_hash_tree_s = b"".join(block_hash_tree)
 
     offsets = pack_offsets(len(verification_key),
                            len(signature),
@@ -192,14 +193,14 @@ def pack_share(prefix, verification_key, signature,
                            len(block_hash_tree_s),
                            len(share_data),
                            len(encprivkey))
-    final_share = "".join([prefix,
-                           offsets,
-                           verification_key,
-                           signature,
-                           share_hash_chain_s,
-                           block_hash_tree_s,
-                           share_data,
-                           encprivkey])
+    final_share = b"".join([prefix,
+                            offsets,
+                            verification_key,
+                            signature,
+                            share_hash_chain_s,
+                            block_hash_tree_s,
+                            share_data,
+                            encprivkey])
     return final_share
 
 def pack_prefix(seqnum, root_hash, IV,
@@ -255,7 +256,7 @@ class SDMFSlotWriteProxy(object):
                                                        self._required_shares)
         assert expected_segment_size == segment_size
 
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
 
         # This is meant to mimic how SDMF files were built before MDMF
         # entered the picture: we generate each share in its entirety,
@@ -343,7 +344,7 @@ class SDMFSlotWriteProxy(object):
             assert len(h) == HASH_SIZE
 
         # serialize the blockhashes, then set them.
-        blockhashes_s = "".join(blockhashes)
+        blockhashes_s = b"".join(blockhashes)
         self._share_pieces['block_hash_tree'] = blockhashes_s
 
         return defer.succeed(None)
@@ -354,12 +355,12 @@ class SDMFSlotWriteProxy(object):
         Add the share hash chain to the share.
         """
         assert isinstance(sharehashes, dict)
-        for h in sharehashes.itervalues():
+        for h in sharehashes.values():
             assert len(h) == HASH_SIZE
 
         # serialize the sharehashes, then set them.
-        sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
-                                 for i in sorted(sharehashes.keys())])
+        sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
+                                  for i in sorted(sharehashes.keys())])
         self._share_pieces['share_hash_chain'] = sharehashes_s
 
         return defer.succeed(None)
@@ -383,7 +384,7 @@ class SDMFSlotWriteProxy(object):
         assert len(salt) == SALT_SIZE
 
         self._share_pieces['salt'] = salt
-        self._share_pieces['sharedata'] = ""
+        self._share_pieces['sharedata'] = b""
 
 
     def get_signable(self):
@@ -519,14 +520,14 @@ class SDMFSlotWriteProxy(object):
         # to the remote server in one write.
         offsets = self._pack_offsets()
         prefix = self.get_signable()
-        final_share = "".join([prefix,
-                               offsets,
-                               self._share_pieces['verification_key'],
-                               self._share_pieces['signature'],
-                               self._share_pieces['share_hash_chain'],
-                               self._share_pieces['block_hash_tree'],
-                               self._share_pieces['sharedata'],
-                               self._share_pieces['encprivkey']])
+        final_share = b"".join([prefix,
+                                offsets,
+                                self._share_pieces['verification_key'],
+                                self._share_pieces['signature'],
+                                self._share_pieces['share_hash_chain'],
+                                self._share_pieces['block_hash_tree'],
+                                self._share_pieces['sharedata'],
+                                self._share_pieces['encprivkey']])
 
         # Our only data vector is going to be writing the final share,
         # in its entirely.
@@ -788,7 +789,7 @@ class MDMFSlotWriteProxy(object):
         # and also because it provides a useful amount of bounds checking.
         self._num_segments = mathutil.div_ceil(self._data_length,
                                                self._segment_size)
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
         # We also calculate the share size, to help us with block
         # constraints later.
         tail_size = self._data_length % self._segment_size
@@ -797,7 +798,7 @@ class MDMFSlotWriteProxy(object):
         else:
             self._tail_block_size = mathutil.next_multiple(tail_size,
                                                            self._required_shares)
-            self._tail_block_size /= self._required_shares
+            self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
 
         # We already know where the sharedata starts; right after the end
         # of the header (which is defined as the signable part + the offsets)
@@ -868,7 +869,7 @@ class MDMFSlotWriteProxy(object):
         else:
             checkstring = seqnum_or_checkstring
 
-        if checkstring == "":
+        if checkstring == b"":
             # We special-case this, since len("") = 0, but we need
             # length of 1 for the case of an empty share to work on the
             # storage server, which is what a checkstring that is the
@@ -876,7 +877,7 @@ class MDMFSlotWriteProxy(object):
             self._testvs = []
         else:
             self._testvs = []
-            self._testvs.append((0, len(checkstring), "eq", checkstring))
+            self._testvs.append((0, len(checkstring), b"eq", checkstring))
 
 
     def __repr__(self):
@@ -893,7 +894,7 @@ class MDMFSlotWriteProxy(object):
         if self._root_hash:
             roothash = self._root_hash
         else:
-            roothash = "\x00" * 32
+            roothash = b"\x00" * 32
         return struct.pack(MDMFCHECKSTRING,
                            1,
                            self._seqnum,
@@ -964,7 +965,7 @@ class MDMFSlotWriteProxy(object):
 
         assert isinstance(blockhashes, list)
 
-        blockhashes_s = "".join(blockhashes)
+        blockhashes_s = b"".join(blockhashes)
         self._offsets['EOF'] = self._offsets['block_hash_tree'] + len(blockhashes_s)
 
         self._writevs.append(tuple([self._offsets['block_hash_tree'],
@@ -998,7 +999,7 @@ class MDMFSlotWriteProxy(object):
         if "verification_key" in self._offsets:
             raise LayoutInvalid("You must write the share hash chain "
                                 "before you write the signature")
-        sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
+        sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
                                   for i in sorted(sharehashes.keys())])
         self._offsets['signature'] = self._offsets['share_hash_chain'] + \
             len(sharehashes_s)
@@ -1149,7 +1150,7 @@ class MDMFSlotWriteProxy(object):
         tw_vectors = {}
         if not self._testvs:
             self._testvs = []
-            self._testvs.append(tuple([0, 1, "eq", ""]))
+            self._testvs.append(tuple([0, 1, b"eq", b""]))
         if not self._written:
             # Write a new checkstring to the share when we write it, so
             # that we have something to check later.
@@ -1157,7 +1158,7 @@ class MDMFSlotWriteProxy(object):
             datavs.append((0, new_checkstring))
             def _first_write():
                 self._written = True
-                self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)]
+                self._testvs = [(0, len(new_checkstring), b"eq", new_checkstring)]
             on_success = _first_write
         tw_vectors[self.shnum] = (self._testvs, datavs, None)
         d = self._storage_server.slot_testv_and_readv_and_writev(
@@ -1317,7 +1318,7 @@ class MDMFSlotReadProxy(object):
         self._segment_size = segsize
         self._data_length = datalen
 
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
         # We can upload empty files, and need to account for this fact
         # so as to avoid zero-division and zero-modulo errors.
         if datalen > 0:
@@ -1329,7 +1330,7 @@ class MDMFSlotReadProxy(object):
         else:
             self._tail_block_size = mathutil.next_multiple(tail_size,
                                                     self._required_shares)
-            self._tail_block_size /= self._required_shares
+            self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
 
         return encoding_parameters
 
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 639780846..6208af0fa 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1451,22 +1451,22 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         self.ss = self.create("MDMFProxies storage test server")
         self.rref = RemoteBucket(self.ss)
         self.storage_server = _StorageServer(lambda: self.rref)
-        self.secrets = (self.write_enabler("we_secret"),
-                        self.renew_secret("renew_secret"),
-                        self.cancel_secret("cancel_secret"))
-        self.segment = "aaaaaa"
-        self.block = "aa"
-        self.salt = "a" * 16
-        self.block_hash = "a" * 32
-        self.block_hash_tree = [self.block_hash for i in xrange(6)]
+        self.secrets = (self.write_enabler(b"we_secret"),
+                        self.renew_secret(b"renew_secret"),
+                        self.cancel_secret(b"cancel_secret"))
+        self.segment = b"aaaaaa"
+        self.block = b"aa"
+        self.salt = b"a" * 16
+        self.block_hash = b"a" * 32
+        self.block_hash_tree = [self.block_hash for i in range(6)]
         self.share_hash = self.block_hash
-        self.share_hash_chain = dict([(i, self.share_hash) for i in xrange(6)])
-        self.signature = "foobarbaz"
-        self.verification_key = "vvvvvv"
-        self.encprivkey = "private"
+        self.share_hash_chain = dict([(i, self.share_hash) for i in range(6)])
+        self.signature = b"foobarbaz"
+        self.verification_key = b"vvvvvv"
+        self.encprivkey = b"private"
         self.root_hash = self.block_hash
         self.salt_hash = self.root_hash
-        self.salt_hash_tree = [self.salt_hash for i in xrange(6)]
+        self.salt_hash_tree = [self.salt_hash for i in range(6)]
         self.block_hash_tree_s = self.serialize_blockhashes(self.block_hash_tree)
         self.share_hash_chain_s = self.serialize_sharehashes(self.share_hash_chain)
         # blockhashes and salt hashes are serialized in the same way,
@@ -1481,15 +1481,19 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def write_enabler(self, we_tag):
-        return hashutil.tagged_hash("we_blah", we_tag)
+        return hashutil.tagged_hash(b"we_blah", we_tag)
 
 
     def renew_secret(self, tag):
-        return hashutil.tagged_hash("renew_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % tag
+        return hashutil.tagged_hash(b"renew_blah", tag)
 
 
     def cancel_secret(self, tag):
-        return hashutil.tagged_hash("cancel_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % tag
+        return hashutil.tagged_hash(b"cancel_blah", tag)
 
 
     def workdir(self, name):
@@ -1531,14 +1535,14 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                                 6,
                                 36)
         # Now we'll build the offsets.
-        sharedata = ""
+        sharedata = b""
         if not tail_segment and not empty:
-            for i in xrange(6):
+            for i in range(6):
                 sharedata += self.salt + self.block
         elif tail_segment:
-            for i in xrange(5):
+            for i in range(5):
                 sharedata += self.salt + self.block
-            sharedata += self.salt + "a"
+            sharedata += self.salt + b"a"
 
         # The encrypted private key comes after the shares + salts
         offset_size = struct.calcsize(MDMFOFFSETS)
@@ -1592,7 +1596,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # and the verification key
         data += self.verification_key
         # Then we'll add in gibberish until we get to the right point.
-        nulls = b"".join([b" " for i in xrange(len(data), share_data_offset)])
+        nulls = b"".join([b" " for i in range(len(data), share_data_offset)])
         data += nulls
 
         # Then the share data
@@ -1626,11 +1630,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def build_test_sdmf_share(self, empty=False):
         if empty:
-            sharedata = ""
+            sharedata = b""
         else:
             sharedata = self.segment * 6
         self.sharedata = sharedata
-        blocksize = len(sharedata) / 3
+        blocksize = len(sharedata) // 3
         block = sharedata[:blocksize]
         self.blockdata = block
         prefix = struct.pack(">BQ32s16s BBQQ",
@@ -1691,7 +1695,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # Check that every method equals what we expect it to.
         d = defer.succeed(None)
@@ -1700,7 +1704,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnlessEqual(block, self.block)
             self.failUnlessEqual(salt, self.salt)
 
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mr.get_block_and_salt(i))
             d.addCallback(_check_block_and_salt)
@@ -1763,7 +1767,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read_with_different_tail_segment_size(self):
-        self.write_test_share_to_server("si1", tail_segment=True)
+        self.write_test_share_to_server(b"si1", tail_segment=True)
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_block_and_salt(5)
         def _check_tail_segment(results):
@@ -1775,7 +1779,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_block_with_invalid_segnum(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -1786,7 +1790,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_encoding_parameters_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_encoding_parameters()
         def _check_encoding_parameters(args):
@@ -1800,7 +1804,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_seqnum_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_seqnum()
         d.addCallback(lambda seqnum:
@@ -1809,7 +1813,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_root_hash_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_root_hash()
         d.addCallback(lambda root_hash:
@@ -1818,7 +1822,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_checkstring_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_checkstring()
         d.addCallback(lambda checkstring:
@@ -1832,9 +1836,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # the test vectors failed, this read vector can help us to
         # diagnose the problem. This test ensures that the read vector
         # is working appropriately.
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
 
-        for i in xrange(6):
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
         mw.put_encprivkey(self.encprivkey)
         mw.put_blockhashes(self.block_hash_tree)
@@ -1849,7 +1853,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnless(result)
             self.failIf(readv)
             self.old_checkstring = mw.get_checkstring()
-            mw.set_checkstring("")
+            mw.set_checkstring(b"")
         d.addCallback(_then)
         d.addCallback(lambda ignored:
             mw.finish_publishing())
@@ -1866,9 +1870,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_private_key_after_share_hash_chain(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -1885,10 +1889,10 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_signature_after_verification_key(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         d = defer.succeed(None)
         # Put everything up to and including the verification key.
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -1915,8 +1919,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # Make two mutable writers, both pointing to the same storage
         # server, both at the same storage index, and try writing to the
         # same share.
-        mw1 = self._make_new_mw("si1", 0)
-        mw2 = self._make_new_mw("si1", 0)
+        mw1 = self._make_new_mw(b"si1", 0)
+        mw2 = self._make_new_mw(b"si1", 0)
 
         def _check_success(results):
             result, readvs = results
@@ -1927,7 +1931,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failIf(result)
 
         def _write_share(mw):
-            for i in xrange(6):
+            for i in range(6):
                 mw.put_block(self.block, i, self.salt)
             mw.put_encprivkey(self.encprivkey)
             mw.put_blockhashes(self.block_hash_tree)
@@ -1947,9 +1951,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_invalid_salt_size(self):
         # Salts need to be 16 bytes in size. Writes that attempt to
         # write more or less than this should be rejected.
-        mw = self._make_new_mw("si1", 0)
-        invalid_salt = "a" * 17 # 17 bytes
-        another_invalid_salt = "b" * 15 # 15 bytes
+        mw = self._make_new_mw(b"si1", 0)
+        invalid_salt = b"a" * 17 # 17 bytes
+        another_invalid_salt = b"b" * 15 # 15 bytes
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "salt too big",
@@ -1977,9 +1981,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             res, d = results
             self.failUnless(results)
 
-        mw = self._make_new_mw("si1", 0)
-        mw.set_checkstring("this is a lie")
-        for i in xrange(6):
+        mw = self._make_new_mw(b"si1", 0)
+        mw.set_checkstring(b"this is a lie")
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
         mw.put_encprivkey(self.encprivkey)
         mw.put_blockhashes(self.block_hash_tree)
@@ -1990,7 +1994,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         d = mw.finish_publishing()
         d.addCallback(_check_failure)
         d.addCallback(lambda ignored:
-            mw.set_checkstring(""))
+            mw.set_checkstring(b""))
         d.addCallback(lambda ignored:
             mw.finish_publishing())
         d.addCallback(_check_success)
@@ -2010,7 +2014,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write(self):
         # This translates to a file with 6 6-byte segments, and with 2-byte
         # blocks.
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         # Test writing some blocks.
         read = self.ss.remote_slot_readv
         expected_private_key_offset = struct.calcsize(MDMFHEADER)
@@ -2021,7 +2025,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                                     SHARE_HASH_CHAIN_SIZE
         written_block_size = 2 + len(self.salt)
         written_block = self.block + self.salt
-        for i in xrange(6):
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
 
         mw.put_encprivkey(self.encprivkey)
@@ -2035,35 +2039,35 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnlessEqual(len(results), 2)
             result, ign = results
             self.failUnless(result, "publish failed")
-            for i in xrange(6):
-                self.failUnlessEqual(read("si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]),
+            for i in range(6):
+                self.failUnlessEqual(read(b"si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]),
                                 {0: [written_block]})
 
             self.failUnlessEqual(len(self.encprivkey), 7)
-            self.failUnlessEqual(read("si1", [0], [(expected_private_key_offset, 7)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_private_key_offset, 7)]),
                                  {0: [self.encprivkey]})
 
             expected_block_hash_offset = expected_sharedata_offset + \
                         (6 * written_block_size)
             self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6)
-            self.failUnlessEqual(read("si1", [0], [(expected_block_hash_offset, 32 * 6)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_block_hash_offset, 32 * 6)]),
                                  {0: [self.block_hash_tree_s]})
 
             expected_share_hash_offset = expected_private_key_offset + len(self.encprivkey)
-            self.failUnlessEqual(read("si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]),
+            self.failUnlessEqual(read(b"si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]),
                                  {0: [self.share_hash_chain_s]})
 
-            self.failUnlessEqual(read("si1", [0], [(9, 32)]),
+            self.failUnlessEqual(read(b"si1", [0], [(9, 32)]),
                                  {0: [self.root_hash]})
             expected_signature_offset = expected_share_hash_offset + \
                 len(self.share_hash_chain_s)
             self.failUnlessEqual(len(self.signature), 9)
-            self.failUnlessEqual(read("si1", [0], [(expected_signature_offset, 9)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_signature_offset, 9)]),
                                  {0: [self.signature]})
 
             expected_verification_key_offset = expected_signature_offset + len(self.signature)
             self.failUnlessEqual(len(self.verification_key), 6)
-            self.failUnlessEqual(read("si1", [0], [(expected_verification_key_offset, 6)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_verification_key_offset, 6)]),
                                  {0: [self.verification_key]})
 
             signable = mw.get_signable()
@@ -2082,49 +2086,49 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
             # Check the version number to make sure that it is correct.
             expected_version_number = struct.pack(">B", 1)
-            self.failUnlessEqual(read("si1", [0], [(0, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(0, 1)]),
                                  {0: [expected_version_number]})
             # Check the sequence number to make sure that it is correct
             expected_sequence_number = struct.pack(">Q", 0)
-            self.failUnlessEqual(read("si1", [0], [(1, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(1, 8)]),
                                  {0: [expected_sequence_number]})
             # Check that the encoding parameters (k, N, segement size, data
             # length) are what they should be. These are  3, 10, 6, 36
             expected_k = struct.pack(">B", 3)
-            self.failUnlessEqual(read("si1", [0], [(41, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(41, 1)]),
                                  {0: [expected_k]})
             expected_n = struct.pack(">B", 10)
-            self.failUnlessEqual(read("si1", [0], [(42, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(42, 1)]),
                                  {0: [expected_n]})
             expected_segment_size = struct.pack(">Q", 6)
-            self.failUnlessEqual(read("si1", [0], [(43, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(43, 8)]),
                                  {0: [expected_segment_size]})
             expected_data_length = struct.pack(">Q", 36)
-            self.failUnlessEqual(read("si1", [0], [(51, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(51, 8)]),
                                  {0: [expected_data_length]})
             expected_offset = struct.pack(">Q", expected_private_key_offset)
-            self.failUnlessEqual(read("si1", [0], [(59, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(59, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_share_hash_offset)
-            self.failUnlessEqual(read("si1", [0], [(67, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(67, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_signature_offset)
-            self.failUnlessEqual(read("si1", [0], [(75, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(75, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_verification_key_offset)
-            self.failUnlessEqual(read("si1", [0], [(83, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(83, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_verification_key_offset + len(self.verification_key))
-            self.failUnlessEqual(read("si1", [0], [(91, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(91, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_sharedata_offset)
-            self.failUnlessEqual(read("si1", [0], [(99, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(99, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_block_hash_offset)
-            self.failUnlessEqual(read("si1", [0], [(107, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(107, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_eof_offset)
-            self.failUnlessEqual(read("si1", [0], [(115, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(115, 8)]),
                                  {0: [expected_offset]})
         d.addCallback(_check_publish)
         return d
@@ -2140,13 +2144,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_write_rejected_with_too_many_blocks(self):
-        mw = self._make_new_mw("si0", 0)
+        mw = self._make_new_mw(b"si0", 0)
 
         # Try writing too many blocks. We should not be able to write
         # more than 6
         # blocks into each share.
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2159,8 +2163,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write_rejected_with_invalid_salt(self):
         # Try writing an invalid salt. Salts are 16 bytes -- any more or
         # less should cause an error.
-        mw = self._make_new_mw("si1", 0)
-        bad_salt = "a" * 17 # 17 bytes
+        mw = self._make_new_mw(b"si1", 0)
+        bad_salt = b"a" * 17 # 17 bytes
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "test_invalid_salt",
@@ -2171,15 +2175,15 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write_rejected_with_invalid_root_hash(self):
         # Try writing an invalid root hash. This should be SHA256d, and
         # 32 bytes long as a result.
-        mw = self._make_new_mw("si2", 0)
+        mw = self._make_new_mw(b"si2", 0)
         # 17 bytes != 32 bytes
-        invalid_root_hash = "a" * 17
+        invalid_root_hash = b"a" * 17
         d = defer.succeed(None)
         # Before this test can work, we need to put some blocks + salts,
         # a block hash tree, and a share hash tree. Otherwise, we'll see
         # failures that match what we are looking for, but are caused by
         # the constraints imposed on operation ordering.
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2199,8 +2203,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # _make_new_mw is 2bytes -- any more or any less than this
         # should be cause for failure, unless it is the tail segment, in
         # which case it may not be failure.
-        invalid_block = "a"
-        mw = self._make_new_mw("si3", 0, 33) # implies a tail segment with
+        invalid_block = b"a"
+        mw = self._make_new_mw(b"si3", 0, 33) # implies a tail segment with
                                              # one byte blocks
         # 1 bytes != 2 bytes
         d = defer.succeed(None)
@@ -2214,7 +2218,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.shouldFail(LayoutInvalid, "test blocksize too large",
                             None,
                             mw.put_block, invalid_block, 0, self.salt))
-        for i in xrange(5):
+        for i in range(5):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         # Try to put an invalid tail segment
@@ -2222,7 +2226,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.shouldFail(LayoutInvalid, "test invalid tail segment",
                             None,
                             mw.put_block, self.block, 5, self.salt))
-        valid_block = "a"
+        valid_block = b"a"
         d.addCallback(lambda ignored:
             mw.put_block(valid_block, 5, self.salt))
         return d
@@ -2247,10 +2251,10 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         #  - share hashes and block hashes before root hash
         #  - root hash before signature
         #  - signature before verification key
-        mw0 = self._make_new_mw("si0", 0)
+        mw0 = self._make_new_mw(b"si0", 0)
         # Write some shares
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw0.put_block(self.block, i, self.salt))
 
@@ -2315,11 +2319,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_end_to_end(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         # Write a share using the mutable writer, and make sure that the
         # reader knows how to read everything back to us.
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2337,13 +2341,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         d.addCallback(lambda ignored:
             mw.finish_publishing())
 
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         def _check_block_and_salt(block_and_salt):
             (block, salt) = block_and_salt
             self.failUnlessEqual(block, self.block)
             self.failUnlessEqual(salt, self.salt)
 
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mr.get_block_and_salt(i))
             d.addCallback(_check_block_and_salt)
@@ -2404,7 +2408,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # The MDMFSlotReadProxy should also know how to read SDMF files,
         # since it will encounter them on the grid. Callers use the
         # is_sdmf method to test this.
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = mr.is_sdmf()
         d.addCallback(lambda issdmf:
@@ -2415,7 +2419,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_reads_sdmf(self):
         # The slot read proxy should, naturally, know how to tell us
         # about data in the SDMF format
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2486,7 +2490,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # SDMF shares have only one segment, so it doesn't make sense to
         # read more segments than that. The reader should know this and
         # complain if we try to do that.
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2507,7 +2511,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # finding out which shares are on the remote peer so that it
         # doesn't waste round trips.
         mdmf_data = self.build_test_mdmf_share()
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         def _make_mr(ignored, length):
             mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, mdmf_data[:length])
             return mr
@@ -2568,7 +2572,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_prefetched_sdmf_data(self):
         sdmf_data = self.build_test_sdmf_share()
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         def _make_mr(ignored, length):
             mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, sdmf_data[:length])
             return mr
@@ -2635,7 +2639,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # Some tests upload a file with no contents to test things
         # unrelated to the actual handling of the content of the file.
         # The reader should behave intelligently in these cases.
-        self.write_test_share_to_server("si1", empty=True)
+        self.write_test_share_to_server(b"si1", empty=True)
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct.
@@ -2661,7 +2665,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read_with_empty_sdmf_file(self):
-        self.write_sdmf_share_to_server("si1", empty=True)
+        self.write_sdmf_share_to_server(b"si1", empty=True)
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct
@@ -2687,7 +2691,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_verinfo_with_sdmf_file(self):
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the version information.
         d = defer.succeed(None)
@@ -2728,7 +2732,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_verinfo_with_mdmf_file(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2804,7 +2808,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         def _then(ignored):
             self.failUnlessEqual(self.rref.write_count, 1)
             read = self.ss.remote_slot_readv
-            self.failUnlessEqual(read("si1", [0], [(0, len(data))]),
+            self.failUnlessEqual(read(b"si1", [0], [(0, len(data))]),
                                  {0: [data]})
         d.addCallback(_then)
         return d
@@ -2812,7 +2816,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_sdmf_writer_preexisting_share(self):
         data = self.build_test_sdmf_share()
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
 
         # Now there is a share on the storage server. To successfully
         # write, we need to set the checkstring correctly. When we
@@ -2861,9 +2865,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         def _then_again(results):
             self.failUnless(results[0])
             read = self.ss.remote_slot_readv
-            self.failUnlessEqual(read("si1", [0], [(1, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(1, 8)]),
                                  {0: [struct.pack(">Q", 1)]})
-            self.failUnlessEqual(read("si1", [0], [(9, len(data) - 9)]),
+            self.failUnlessEqual(read(b"si1", [0], [(9, len(data) - 9)]),
                                  {0: [data[9:]]})
         d.addCallback(_then_again)
         return d

From 2ba0854e0d80abedb1e5e9d8fc9fc6fc1af746cf Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 12:59:03 -0400
Subject: [PATCH 44/59] More passing tests.

---
 src/allmydata/mutable/layout.py    | 14 +++++++-------
 src/allmydata/test/test_storage.py | 24 ++++++++++++------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index 5f459921a..ee086ec5e 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -297,7 +297,7 @@ class SDMFSlotWriteProxy(object):
                                       salt)
         else:
             checkstring = checkstring_or_seqnum
-        self._testvs = [(0, len(checkstring), "eq", checkstring)]
+        self._testvs = [(0, len(checkstring), b"eq", checkstring)]
 
 
     def get_checkstring(self):
@@ -307,7 +307,7 @@ class SDMFSlotWriteProxy(object):
         """
         if self._testvs:
             return self._testvs[0][3]
-        return ""
+        return b""
 
 
     def put_block(self, data, segnum, salt):
@@ -538,7 +538,7 @@ class SDMFSlotWriteProxy(object):
             # yet, so we assume that we are writing a new share, and set
             # a test vector that will allow a new share to be written.
             self._testvs = []
-            self._testvs.append(tuple([0, 1, "eq", ""]))
+            self._testvs.append(tuple([0, 1, b"eq", b""]))
 
         tw_vectors = {}
         tw_vectors[self.shnum] = (self._testvs, datavs, None)
@@ -1195,7 +1195,7 @@ class MDMFSlotReadProxy(object):
                  storage_server,
                  storage_index,
                  shnum,
-                 data="",
+                 data=b"",
                  data_is_everything=False):
         # Start the initialization process.
         self._storage_server = storage_server
@@ -1239,7 +1239,7 @@ class MDMFSlotReadProxy(object):
         # None if there isn't any cached data, but the way we index the
         # cached data requires a string, so convert None to "".
         if self._data == None:
-            self._data = ""
+            self._data = b""
 
 
     def _maybe_fetch_offsets_and_header(self, force_remote=False):
@@ -1417,7 +1417,7 @@ class MDMFSlotReadProxy(object):
                 # when we fetched the header
                 data = results[self.shnum]
                 if not data:
-                    data = ""
+                    data = b""
                 else:
                     if len(data) != 1:
                         raise BadShareError("got %d vectors, not 1" % len(data))
@@ -1426,7 +1426,7 @@ class MDMFSlotReadProxy(object):
             else:
                 data = results[self.shnum]
                 if not data:
-                    salt = data = ""
+                    salt = data = b""
                 else:
                     salt_and_data = results[self.shnum][0]
                     salt = salt_and_data[:SALT_SIZE]
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 6208af0fa..df625eda3 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -2409,7 +2409,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # since it will encounter them on the grid. Callers use the
         # is_sdmf method to test this.
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.is_sdmf()
         d.addCallback(lambda issdmf:
             self.failUnless(issdmf))
@@ -2420,7 +2420,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # The slot read proxy should, naturally, know how to tell us
         # about data in the SDMF format
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.is_sdmf())
@@ -2491,7 +2491,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # read more segments than that. The reader should know this and
         # complain if we try to do that.
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.is_sdmf())
@@ -2513,7 +2513,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         mdmf_data = self.build_test_mdmf_share()
         self.write_test_share_to_server(b"si1")
         def _make_mr(ignored, length):
-            mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, mdmf_data[:length])
+            mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0, mdmf_data[:length])
             return mr
 
         d = defer.succeed(None)
@@ -2574,7 +2574,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         sdmf_data = self.build_test_sdmf_share()
         self.write_sdmf_share_to_server(b"si1")
         def _make_mr(ignored, length):
-            mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, sdmf_data[:length])
+            mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0, sdmf_data[:length])
             return mr
 
         d = defer.succeed(None)
@@ -2640,7 +2640,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # unrelated to the actual handling of the content of the file.
         # The reader should behave intelligently in these cases.
         self.write_test_share_to_server(b"si1", empty=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct.
         d = defer.succeed(None)
@@ -2666,7 +2666,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_empty_sdmf_file(self):
         self.write_sdmf_share_to_server(b"si1", empty=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct
         d = defer.succeed(None)
@@ -2692,7 +2692,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_verinfo_with_sdmf_file(self):
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the version information.
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2733,7 +2733,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_verinfo_with_mdmf_file(self):
         self.write_test_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.get_verinfo())
@@ -2780,7 +2780,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         data = self.build_test_sdmf_share()
         sdmfr = SDMFSlotWriteProxy(0,
                                    self.storage_server,
-                                   "si1",
+                                   b"si1",
                                    self.secrets,
                                    0, 3, 10, 36, 36)
         # Put the block and salt.
@@ -2823,7 +2823,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # don't, no write should occur.
         sdmfw = SDMFSlotWriteProxy(0,
                                    self.storage_server,
-                                   "si1",
+                                   b"si1",
                                    self.secrets,
                                    1, 3, 10, 36, 36)
         sdmfw.put_block(self.blockdata, 0, self.salt)
@@ -2845,7 +2845,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         sdmfw.put_verification_key(self.verification_key)
 
         # We shouldn't have a checkstring yet
-        self.failUnlessEqual(sdmfw.get_checkstring(), "")
+        self.failUnlessEqual(sdmfw.get_checkstring(), b"")
 
         d = sdmfw.finish_publishing()
         def _then(results):

From 6c52a03030dcf67d22babd6304225aaee818fd35 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 11:59:34 -0400
Subject: [PATCH 45/59] Fix indent.

---
 src/allmydata/test/test_storage.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index df625eda3..3fc4fd11f 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1662,13 +1662,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                               encprivkey_offset,
                               eof_offset)
         final_share = b"".join([prefix,
-                           offsets,
-                           self.verification_key,
-                           self.signature,
-                           self.share_hash_chain_s,
-                           self.block_hash_tree_s,
-                           block,
-                           self.encprivkey])
+                                offsets,
+                                self.verification_key,
+                                self.signature,
+                                self.share_hash_chain_s,
+                                self.block_hash_tree_s,
+                                block,
+                                self.encprivkey])
         self.offsets = {}
         self.offsets['signature'] = signature_offset
         self.offsets['share_hash_chain'] = sharehashes_offset

From 2b37da9ca027ade1d22e1a582d894600dbb6d884 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 11:59:39 -0400
Subject: [PATCH 46/59] filter() is lazy in Python 3.

---
 src/allmydata/mutable/layout.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index ee086ec5e..bf9a0483b 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -1744,7 +1744,7 @@ class MDMFSlotReadProxy(object):
 
 
     def _read(self, readvs, force_remote=False):
-        unsatisfiable = filter(lambda x: x[0] + x[1] > len(self._data), readvs)
+        unsatisfiable = list(filter(lambda x: x[0] + x[1] > len(self._data), readvs))
         # TODO: It's entirely possible to tweak this so that it just
         # fulfills the requests that it can, and not demand that all
         # requests are satisfiable before running it.

From 1cfe58a52dfae9682cb01aee8e8f077e919796be Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 13:17:52 -0400
Subject: [PATCH 47/59] All of test_storage passes on Python 3.

---
 src/allmydata/storage/immutable.py |  8 ++++++--
 src/allmydata/storage/server.py    | 11 ++++++++---
 src/allmydata/test/test_storage.py |  8 ++++----
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py
index 32ba1a739..8d83ec1b3 100644
--- a/src/allmydata/storage/immutable.py
+++ b/src/allmydata/storage/immutable.py
@@ -1,3 +1,5 @@
+from future.utils import bytes_to_native_str
+
 import os, stat, struct, time
 
 from foolscap.api import Referenceable
@@ -298,7 +300,9 @@ class BucketReader(Referenceable):
 
     def __repr__(self):
         return "<%s %s %s>" % (self.__class__.__name__,
-                               base32.b2a(self.storage_index[:8])[:12],
+                               bytes_to_native_str(
+                                   base32.b2a(self.storage_index[:8])[:12]
+                               ),
                                self.shnum)
 
     def remote_read(self, offset, length):
@@ -309,7 +313,7 @@ class BucketReader(Referenceable):
         return data
 
     def remote_advise_corrupt_share(self, reason):
-        return self.ss.remote_advise_corrupt_share("immutable",
+        return self.ss.remote_advise_corrupt_share(b"immutable",
                                                    self.storage_index,
                                                    self.shnum,
                                                    reason)
diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 387225b01..ad8001a60 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -1,3 +1,4 @@
+from future.utils import bytes_to_native_str
 import os, re, struct, time
 import weakref
 import six
@@ -676,6 +677,10 @@ class StorageServer(service.MultiService, Referenceable):
 
     def remote_advise_corrupt_share(self, share_type, storage_index, shnum,
                                     reason):
+        # This is a remote API, I believe, so this has to be bytes for legacy
+        # protocol backwards compatibility reasons.
+        assert isinstance(share_type, bytes)
+        assert isinstance(reason, bytes)
         fileutil.make_dirs(self.corruption_advisory_dir)
         now = time_format.iso_utc(sep="T")
         si_s = si_b2a(storage_index)
@@ -684,11 +689,11 @@ class StorageServer(service.MultiService, Referenceable):
                           "%s--%s-%d" % (now, si_s, shnum)).replace(":","")
         with open(fn, "w") as f:
             f.write("report: Share Corruption\n")
-            f.write("type: %s\n" % share_type)
-            f.write("storage_index: %s\n" % si_s)
+            f.write("type: %s\n" % bytes_to_native_str(share_type))
+            f.write("storage_index: %s\n" % bytes_to_native_str(si_s))
             f.write("share_number: %d\n" % shnum)
             f.write("\n")
-            f.write(reason)
+            f.write(bytes_to_native_str(reason))
             f.write("\n")
         log.msg(format=("client claims corruption in (%(share_type)s) " +
                         "%(si)s-%(shnum)d: %(reason)s"),
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 3fc4fd11f..5c8faabc3 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,4 @@
-from future.utils import native_str, PY3
+from future.utils import native_str, PY3, bytes_to_native_str
 
 import time
 import os.path
@@ -731,7 +731,7 @@ class Server(unittest.TestCase):
 
         si0_s = base32.b2a(b"si0")
         ss.remote_advise_corrupt_share(b"immutable", b"si0", 0,
-                                       "This share smells funny.\n")
+                                       b"This share smells funny.\n")
         reportdir = os.path.join(workdir, "corruption-advisories")
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 1)
@@ -755,11 +755,11 @@ class Server(unittest.TestCase):
 
         b = ss.remote_get_buckets(b"si1")
         self.failUnlessEqual(set(b.keys()), set([1]))
-        b[1].remote_advise_corrupt_share("This share tastes like dust.\n")
+        b[1].remote_advise_corrupt_share(b"This share tastes like dust.\n")
 
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 2)
-        report_si1 = [r for r in reports if si1_s in r][0]
+        report_si1 = [r for r in reports if bytes_to_native_str(si1_s) in r][0]
         f = open(os.path.join(reportdir, report_si1), "rb")
         report = f.read()
         f.close()

From 3fa919834ad0883e560cabfeb27df579186a0d83 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 13:20:57 -0400
Subject: [PATCH 48/59] Finish port to Python 3.

---
 newsfragments/3397.minor           |  0
 src/allmydata/test/test_storage.py | 16 ++++++++++++----
 src/allmydata/util/_python3.py     |  1 +
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 newsfragments/3397.minor

diff --git a/newsfragments/3397.minor b/newsfragments/3397.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 5c8faabc3..072cc9127 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,12 @@
-from future.utils import native_str, PY3, bytes_to_native_str
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import native_str, PY2, PY3, bytes_to_native_str
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 import time
 import os.path
@@ -392,7 +400,7 @@ class Server(unittest.TestCase):
 
     def test_remove_incoming(self):
         ss = self.create("test_remove_incoming")
-        already, writers = self.allocate(ss, b"vid", range(3), 10)
+        already, writers = self.allocate(ss, b"vid", list(range(3)), 10)
         for i,wb in writers.items():
             wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
@@ -563,7 +571,7 @@ class Server(unittest.TestCase):
 
         # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
         # 5000-1085=3915 free, therefore we can fit 39 100byte shares
-        already3, writers3 = self.allocate(ss, b"vid3", range(100), 100, canary)
+        already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary)
         self.failUnlessEqual(len(writers3), 39)
         self.failUnlessEqual(len(ss._active_writers), 39)
 
@@ -598,7 +606,7 @@ class Server(unittest.TestCase):
     def test_leases(self):
         ss = self.create("test_leases")
         canary = FakeCanary()
-        sharenums = range(5)
+        sharenums = list(range(5))
         size = 100
 
         rs0,cs0 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 683834673..afdbea1f0 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -92,6 +92,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
+    "allmydata.test.test_storage",
     "allmydata.test.test_storage_web",
     "allmydata.test.test_time_format",
     "allmydata.test.test_uri",

From d195ae43232b654371d1f3d476fc0b7bde640be9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 1 Sep 2020 11:43:24 -0400
Subject: [PATCH 49/59] Get rid of second, less lenient variant of
 ShouldFailMixin.

---
 src/allmydata/test/common_py3.py | 63 +++++++++++---------------------
 1 file changed, 22 insertions(+), 41 deletions(-)

diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py
index 36738d559..50fb02ff7 100644
--- a/src/allmydata/test/common_py3.py
+++ b/src/allmydata/test/common_py3.py
@@ -79,6 +79,28 @@ class ShouldFailMixin(object):
 
     def shouldFail(self, expected_failure, which, substring,
                    callable, *args, **kwargs):
+        """Assert that a function call raises some exception. This is a
+        Deferred-friendly version of TestCase.assertRaises() .
+
+        Suppose you want to verify the following function:
+
+         def broken(a, b, c):
+             if a < 0:
+                 raise TypeError('a must not be negative')
+             return defer.succeed(b+c)
+
+        You can use:
+            d = self.shouldFail(TypeError, 'test name',
+                                'a must not be negative',
+                                broken, -4, 5, c=12)
+        in your test method. The 'test name' string will be included in the
+        error message, if any, because Deferred chains frequently make it
+        difficult to tell which assertion was tripped.
+
+        The substring= argument, if not None, must appear in the 'repr'
+        of the message wrapped by this Failure, or the test will fail.
+        """
+
         assert substring is None or isinstance(substring, (bytes, unicode))
         d = defer.maybeDeferred(callable, *args, **kwargs)
         def done(res):
@@ -143,44 +165,3 @@ class LoggingServiceParent(service.MultiService):
     def log(self, *args, **kwargs):
         return log.msg(*args, **kwargs)
 
-
-class ShouldFailMixin(object):
-    def shouldFail(self, expected_failure, which, substring,
-                   callable, *args, **kwargs):
-        """Assert that a function call raises some exception. This is a
-        Deferred-friendly version of TestCase.assertRaises() .
-
-        Suppose you want to verify the following function:
-
-         def broken(a, b, c):
-             if a < 0:
-                 raise TypeError('a must not be negative')
-             return defer.succeed(b+c)
-
-        You can use:
-            d = self.shouldFail(TypeError, 'test name',
-                                'a must not be negative',
-                                broken, -4, 5, c=12)
-        in your test method. The 'test name' string will be included in the
-        error message, if any, because Deferred chains frequently make it
-        difficult to tell which assertion was tripped.
-
-        The substring= argument, if not None, must appear in the 'repr'
-        of the message wrapped by this Failure, or the test will fail.
-        """
-
-        assert substring is None or isinstance(substring, str)
-        d = defer.maybeDeferred(callable, *args, **kwargs)
-        def done(res):
-            if isinstance(res, failure.Failure):
-                res.trap(expected_failure)
-                if substring:
-                    message = repr(res.value.args[0])
-                    self.failUnless(substring in message,
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, message))
-            else:
-                self.fail("%s was supposed to raise %s, not get '%s'" %
-                          (which, expected_failure, res))
-        d.addBoth(done)
-        return d

From 0aec62b122b4f34d692a74c9af9d2fe25d47ab18 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 1 Sep 2020 11:46:47 -0400
Subject: [PATCH 50/59] Fix lint.

---
 src/allmydata/test/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index dbd93df67..b69d58ab9 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -52,7 +52,6 @@ from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.internet.interfaces import IPullProducer
 from twisted.python import failure
 from twisted.python.filepath import FilePath
-from twisted.application import service
 from twisted.web.error import Error as WebError
 from twisted.internet.interfaces import (
     IStreamServerEndpointStringParser,

From 063ee18da33bbe8e53a29498fa0117fd1d394301 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 1 Sep 2020 12:45:32 -0400
Subject: [PATCH 51/59] News fragment.

---
 newsfragments/3396.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3396.minor

diff --git a/newsfragments/3396.minor b/newsfragments/3396.minor
new file mode 100644
index 000000000..e69de29bb

From f5a689f0e085a133a2123e93de15909fb26b38b5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 1 Sep 2020 12:46:16 -0400
Subject: [PATCH 52/59] Extend timeout.

---
 src/allmydata/test/test_system.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index c80aeb45e..96fdfaba7 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -980,6 +980,8 @@ class CountingDataUploadable(upload.Data):
 
 class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
+    timeout = 180
+
     def test_connections(self):
         self.basedir = "system/SystemTest/test_connections"
         d = self.set_up_nodes()

From 38f5a9c5a308944131dd586f63f69a25ab53df45 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 4 Sep 2020 12:05:31 -0400
Subject: [PATCH 53/59] Disable coverage on pypy, in the hopes of making it
 less flaky.

---
 .circleci/config.yml     | 3 ++-
 newsfragments/3401.minor | 0
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/3401.minor

diff --git a/.circleci/config.yml b/.circleci/config.yml
index df181f058..56bc3e07e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -211,7 +211,8 @@ jobs:
 
     environment:
       <<: *UTF_8_ENVIRONMENT
-      TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27-coverage"
+      # We don't do coverage since it makes PyPy far too slow:
+      TAHOE_LAFS_TOX_ENVIRONMENT: "pypy27"
 
 
   c-locale:
diff --git a/newsfragments/3401.minor b/newsfragments/3401.minor
new file mode 100644
index 000000000..e69de29bb

From 2787554a8bb471f98d268f94abe225d85c5a2d68 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 4 Sep 2020 12:05:42 -0400
Subject: [PATCH 54/59] Make sure CircleCI see progress output from subunitv2
 reporter.

---
 .circleci/run-tests.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index 48f500280..764651c40 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -68,6 +68,10 @@ export SUBUNITREPORTER_OUTPUT_PATH="${SUBUNIT2}"
 export TAHOE_LAFS_TRIAL_ARGS="${TAHOE_LAFS_TRIAL_ARGS:---reporter=subunitv2-file --rterrors}"
 export PIP_NO_INDEX="1"
 
+# Make output unbuffered, so progress reports from subunitv2-file get streamed
+# and notify CircleCI we're still alive.
+export PYTHONUNBUFFERED=1
+
 if [ "${ALLOWED_FAILURE}" = "yes" ]; then
     alternative="true"
 else

From 6da338a86a2212b13396f074332133699a57846d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:09:35 -0400
Subject: [PATCH 55/59] Note it's been ported.

---
 src/allmydata/test/test_storage.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 072cc9127..797d02794 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,3 +1,8 @@
+"""
+Tests for allmydata.storage.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

From 69575401569d6fb57d2f8df57a2ae389d0ed50ab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:10:13 -0400
Subject: [PATCH 56/59] Assert nodeid is bytes, to ease porting.

---
 src/allmydata/storage/server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index ad8001a60..c044a9fb0 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -52,6 +52,7 @@ class StorageServer(service.MultiService, Referenceable):
         service.MultiService.__init__(self)
         assert isinstance(nodeid, bytes)
         assert len(nodeid) == 20
+        assert isinstance(nodeid, bytes)
         self.my_nodeid = nodeid
         self.storedir = storedir
         sharedir = os.path.join(storedir, "shares")

From fad93f4144bb2da4d629f6c36d5668628e989db9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:13:22 -0400
Subject: [PATCH 57/59] Use existing utility.

---
 src/allmydata/test/test_storage.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 797d02794..4bbbcda30 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -8,7 +8,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import native_str, PY2, PY3, bytes_to_native_str
+from future.utils import native_str, PY2, bytes_to_native_str
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
@@ -1373,8 +1373,7 @@ class MutableServer(unittest.TestCase):
         si = base32.b2a(b"si1")
         # note: this is a detail of the storage server implementation, and
         # may change in the future
-        if PY3:
-            si = si.decode("utf-8")
+        si = bytes_to_native_str(si)  # filesystem paths are native strings
         prefix = si[:2]
         prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix)
         bucketdir = os.path.join(prefixdir, si)

From 810c7eefd8b9b047476144135a948dc41b47d54a Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 8 Sep 2020 16:31:57 -0400
Subject: [PATCH 58/59] Pin Pyrsistent at a Python 2.7 compatible version

---
 newsfragments/3403.minor | 0
 setup.py                 | 5 +++++
 2 files changed, 5 insertions(+)
 create mode 100644 newsfragments/3403.minor

diff --git a/newsfragments/3403.minor b/newsfragments/3403.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/setup.py b/setup.py
index db5a5490d..4c8ccd2a3 100644
--- a/setup.py
+++ b/setup.py
@@ -116,6 +116,11 @@ install_requires = [
     # know works on Python 2.7.
     "eliot ~= 1.7",
 
+    # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped
+    # Python 2 entirely; stick to the version known to work for us.
+    # XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404
+    "pyrsistent <= 0.16.0",
+
     # A great way to define types of values.
     # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390
     "attrs >= 18.2.0, < 20",

From e893254855c552836edfd5dc8164cf045de5d84a Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Tue, 8 Sep 2020 17:23:08 -0400
Subject: [PATCH 59/59] Use Pyrsistent < 0.17.0

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4c8ccd2a3..215e9f20f 100644
--- a/setup.py
+++ b/setup.py
@@ -119,7 +119,7 @@ install_requires = [
     # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped
     # Python 2 entirely; stick to the version known to work for us.
     # XXX: drop this bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3404
-    "pyrsistent <= 0.16.0",
+    "pyrsistent < 0.17.0",
 
     # A great way to define types of values.
     # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390