From 6007c1f67f3b6350ddf5c34966c236526d90092e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:36:54 -0400
Subject: [PATCH 01/13] Some tests are passing.

---
 src/allmydata/immutable/layout.py  | 10 +--
 src/allmydata/test/test_storage.py | 98 +++++++++++++++---------------
 2 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py
index 320f742e0..5d1f691b9 100644
--- a/src/allmydata/immutable/layout.py
+++ b/src/allmydata/immutable/layout.py
@@ -171,7 +171,7 @@ class WriteBucketProxy(object):
     def put_block(self, segmentnum, data):
         offset = self._offsets['data'] + segmentnum * self._block_size
         assert offset + len(data) <= self._offsets['uri_extension']
-        assert isinstance(data, str)
+        assert isinstance(data, bytes)
         if segmentnum < self._num_segments-1:
             precondition(len(data) == self._block_size,
                          len(data), self._block_size)
@@ -185,7 +185,7 @@ class WriteBucketProxy(object):
     def put_crypttext_hashes(self, hashes):
         offset = self._offsets['crypttext_hash_tree']
         assert isinstance(hashes, list)
-        data = "".join(hashes)
+        data = b"".join(hashes)
         precondition(len(data) == self._segment_hash_size,
                      len(data), self._segment_hash_size)
         precondition(offset + len(data) <= self._offsets['block_hashes'],
@@ -196,7 +196,7 @@ class WriteBucketProxy(object):
     def put_block_hashes(self, blockhashes):
         offset = self._offsets['block_hashes']
         assert isinstance(blockhashes, list)
-        data = "".join(blockhashes)
+        data = b"".join(blockhashes)
         precondition(len(data) == self._segment_hash_size,
                      len(data), self._segment_hash_size)
         precondition(offset + len(data) <= self._offsets['share_hashes'],
@@ -209,7 +209,7 @@ class WriteBucketProxy(object):
         # as 2+32=34 bytes each
         offset = self._offsets['share_hashes']
         assert isinstance(sharehashes, list)
-        data = "".join([struct.pack(">H", hashnum) + hashvalue
+        data = b"".join([struct.pack(">H", hashnum) + hashvalue
                         for hashnum,hashvalue in sharehashes])
         precondition(len(data) == self._share_hashtree_size,
                      len(data), self._share_hashtree_size)
@@ -220,7 +220,7 @@ class WriteBucketProxy(object):
 
     def put_uri_extension(self, data):
         offset = self._offsets['uri_extension']
-        assert isinstance(data, str)
+        assert isinstance(data, bytes)
         precondition(len(data) <= self._uri_extension_size_max,
                      len(data), self._uri_extension_size_max)
         length = struct.pack(self.fieldstruct, len(data))
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index d04e6b83d..9395ff8e7 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -65,32 +65,32 @@ class Bucket(unittest.TestCase):
         cancel_secret = os.urandom(32)
         expiration_time = time.time() + 5000
         return LeaseInfo(owner_num, renew_secret, cancel_secret,
-                         expiration_time, "\x00" * 20)
+                         expiration_time, b"\x00" * 20)
 
     def test_create(self):
         incoming, final = self.make_workdir("test_create")
         bw = BucketWriter(self, incoming, final, 200, self.make_lease(),
                           FakeCanary())
-        bw.remote_write(0, "a"*25)
-        bw.remote_write(25, "b"*25)
-        bw.remote_write(50, "c"*25)
-        bw.remote_write(75, "d"*7)
+        bw.remote_write(0, b"a"*25)
+        bw.remote_write(25, b"b"*25)
+        bw.remote_write(50, b"c"*25)
+        bw.remote_write(75, b"d"*7)
         bw.remote_close()
 
     def test_readwrite(self):
         incoming, final = self.make_workdir("test_readwrite")
         bw = BucketWriter(self, incoming, final, 200, self.make_lease(),
                           FakeCanary())
-        bw.remote_write(0, "a"*25)
-        bw.remote_write(25, "b"*25)
-        bw.remote_write(50, "c"*7) # last block may be short
+        bw.remote_write(0, b"a"*25)
+        bw.remote_write(25, b"b"*25)
+        bw.remote_write(50, b"c"*7) # last block may be short
         bw.remote_close()
 
         # now read from it
         br = BucketReader(self, bw.finalhome)
-        self.failUnlessEqual(br.remote_read(0, 25), "a"*25)
-        self.failUnlessEqual(br.remote_read(25, 25), "b"*25)
-        self.failUnlessEqual(br.remote_read(50, 7), "c"*7)
+        self.failUnlessEqual(br.remote_read(0, 25), b"a"*25)
+        self.failUnlessEqual(br.remote_read(25, 25), b"b"*25)
+        self.failUnlessEqual(br.remote_read(50, 7), b"c"*7)
 
     def test_read_past_end_of_share_data(self):
         # test vector for immutable files (hard-coded contents of an immutable share
@@ -107,12 +107,12 @@ class Bucket(unittest.TestCase):
         # complicated string involving hash trees and a URI Extension Block
         # -- see allmydata/immutable/layout.py . This test, which is
         # simulating a client, just sends 'a'.
-        share_data = 'a'
+        share_data = b'a'
 
         ownernumber = struct.pack('>L', 0)
-        renewsecret  = 'THIS LETS ME RENEW YOUR FILE....'
+        renewsecret  = b'THIS LETS ME RENEW YOUR FILE....'
         assert len(renewsecret) == 32
-        cancelsecret = 'THIS LETS ME KILL YOUR FILE HAHA'
+        cancelsecret = b'THIS LETS ME KILL YOUR FILE HAHA'
         assert len(cancelsecret) == 32
         expirationtime = struct.pack('>L', 60*60*24*31) # 31 days in seconds
 
@@ -184,7 +184,7 @@ class BucketProxy(unittest.TestCase):
         cancel_secret = os.urandom(32)
         expiration_time = time.time() + 5000
         return LeaseInfo(owner_num, renew_secret, cancel_secret,
-                         expiration_time, "\x00" * 20)
+                         expiration_time, b"\x00" * 20)
 
     def bucket_writer_closed(self, bw, consumed):
         pass
@@ -217,13 +217,13 @@ class BucketProxy(unittest.TestCase):
 
         sharesize = header_size + 100 + 7*32 + 7*32 + 7*32 + 3*(2+32) + 4+500
 
-        crypttext_hashes = [hashutil.tagged_hash("crypt", "bar%d" % i)
+        crypttext_hashes = [hashutil.tagged_hash(b"crypt", b"bar%d" % i)
                             for i in range(7)]
-        block_hashes = [hashutil.tagged_hash("block", "bar%d" % i)
+        block_hashes = [hashutil.tagged_hash(b"block", b"bar%d" % i)
                         for i in range(7)]
-        share_hashes = [(i, hashutil.tagged_hash("share", "bar%d" % i))
+        share_hashes = [(i, hashutil.tagged_hash(b"share", b"bar%d" % i))
                         for i in (1,9,13)]
-        uri_extension = "s" + "E"*498 + "e"
+        uri_extension = b"s" + b"E"*498 + b"e"
 
         bw, rb, sharefname = self.make_bucket(name, sharesize)
         bp = wbp_class(rb, None,
@@ -234,10 +234,10 @@ class BucketProxy(unittest.TestCase):
                        uri_extension_size_max=len(uri_extension))
 
         d = bp.put_header()
-        d.addCallback(lambda res: bp.put_block(0, "a"*25))
-        d.addCallback(lambda res: bp.put_block(1, "b"*25))
-        d.addCallback(lambda res: bp.put_block(2, "c"*25))
-        d.addCallback(lambda res: bp.put_block(3, "d"*20))
+        d.addCallback(lambda res: bp.put_block(0, b"a"*25))
+        d.addCallback(lambda res: bp.put_block(1, b"b"*25))
+        d.addCallback(lambda res: bp.put_block(2, b"c"*25))
+        d.addCallback(lambda res: bp.put_block(3, b"d"*20))
         d.addCallback(lambda res: bp.put_crypttext_hashes(crypttext_hashes))
         d.addCallback(lambda res: bp.put_block_hashes(block_hashes))
         d.addCallback(lambda res: bp.put_share_hashes(share_hashes))
@@ -248,19 +248,19 @@ class BucketProxy(unittest.TestCase):
         def _start_reading(res):
             br = BucketReader(self, sharefname)
             rb = RemoteBucket(br)
-            server = NoNetworkServer("abc", None)
-            rbp = rbp_class(rb, server, storage_index="")
+            server = NoNetworkServer(b"abc", None)
+            rbp = rbp_class(rb, server, storage_index=b"")
             self.failUnlessIn("to peer", repr(rbp))
             self.failUnless(interfaces.IStorageBucketReader.providedBy(rbp), rbp)
 
             d1 = rbp.get_block_data(0, 25, 25)
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "a"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"a"*25))
             d1.addCallback(lambda res: rbp.get_block_data(1, 25, 25))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "b"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"b"*25))
             d1.addCallback(lambda res: rbp.get_block_data(2, 25, 25))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "c"*25))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"c"*25))
             d1.addCallback(lambda res: rbp.get_block_data(3, 25, 20))
-            d1.addCallback(lambda res: self.failUnlessEqual(res, "d"*20))
+            d1.addCallback(lambda res: self.failUnlessEqual(res, b"d"*20))
 
             d1.addCallback(lambda res: rbp.get_crypttext_hashes())
             d1.addCallback(lambda res:
@@ -302,7 +302,7 @@ class Server(unittest.TestCase):
 
     def create(self, name, reserved_space=0, klass=StorageServer):
         workdir = self.workdir(name)
-        ss = klass(workdir, "\x00" * 20, reserved_space=reserved_space,
+        ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space,
                    stats_provider=FakeStatsProvider())
         ss.setServiceParent(self.sparent)
         return ss
@@ -330,8 +330,8 @@ class Server(unittest.TestCase):
         self.failUnlessIn('available-space', sv1)
 
     def allocate(self, ss, storage_index, sharenums, size, canary=None):
-        renew_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())
-        cancel_secret = hashutil.tagged_hash("blah", "%d" % self._lease_secret.next())
+        renew_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))
+        cancel_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))
         if not canary:
             canary = FakeCanary()
         return ss.remote_allocate_buckets(storage_index,
@@ -390,7 +390,7 @@ class Server(unittest.TestCase):
 
     def test_remove_incoming(self):
         ss = self.create("test_remove_incoming")
-        already, writers = self.allocate(ss, "vid", range(3), 10)
+        already, writers = self.allocate(ss, b"vid", range(3), 10)
         for i,wb in writers.items():
             wb.remote_write(0, "%10d" % i)
             wb.remote_close()
@@ -524,14 +524,14 @@ class Server(unittest.TestCase):
         OVERHEAD = 3*4
         LEASE_SIZE = 4+32+32+4
         canary = FakeCanary(True)
-        already, writers = self.allocate(ss, "vid1", [0,1,2], 1000, canary)
+        already, writers = self.allocate(ss, b"vid1", [0,1,2], 1000, canary)
         self.failUnlessEqual(len(writers), 3)
         # now the StorageServer should have 3000 bytes provisionally
         # allocated, allowing only 2000 more to be claimed
         self.failUnlessEqual(len(ss._active_writers), 3)
 
         # allocating 1001-byte shares only leaves room for one
-        already2, writers2 = self.allocate(ss, "vid2", [0,1,2], 1001, canary)
+        already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary)
         self.failUnlessEqual(len(writers2), 1)
         self.failUnlessEqual(len(ss._active_writers), 4)
 
@@ -578,19 +578,19 @@ class Server(unittest.TestCase):
         fileutil.make_dirs(basedir)
         filename = os.path.join(basedir, "testfile")
         f = open(filename, "wb")
-        f.write("start")
+        f.write(b"start")
         f.close()
         # mode="w" allows seeking-to-create-holes, but truncates pre-existing
         # files. mode="a" preserves previous contents but does not allow
         # seeking-to-create-holes. mode="r+" allows both.
         f = open(filename, "rb+")
         f.seek(100)
-        f.write("100")
+        f.write(b"100")
         f.close()
         filelen = os.stat(filename)[stat.ST_SIZE]
         self.failUnlessEqual(filelen, 100+3)
         f2 = open(filename, "rb")
-        self.failUnlessEqual(f2.read(5), "start")
+        self.failUnlessEqual(f2.read(5), b"start")
 
 
     def test_leases(self):
@@ -689,10 +689,10 @@ class Server(unittest.TestCase):
 
     def test_readonly(self):
         workdir = self.workdir("test_readonly")
-        ss = StorageServer(workdir, "\x00" * 20, readonly_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, readonly_storage=True)
         ss.setServiceParent(self.sparent)
 
-        already,writers = self.allocate(ss, "vid", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"vid", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(writers, {})
 
@@ -706,25 +706,25 @@ class Server(unittest.TestCase):
     def test_discard(self):
         # discard is really only used for other tests, but we test it anyways
         workdir = self.workdir("test_discard")
-        ss = StorageServer(workdir, "\x00" * 20, discard_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
-        already,writers = self.allocate(ss, "vid", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"vid", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
         for i,wb in writers.items():
-            wb.remote_write(0, "%25d" % i)
+            wb.remote_write(0, b"%25d" % i)
             wb.remote_close()
         # since we discard the data, the shares should be present but sparse.
         # Since we write with some seeks, the data we read back will be all
         # zeros.
-        b = ss.remote_get_buckets("vid")
+        b = ss.remote_get_buckets(b"vid")
         self.failUnlessEqual(set(b.keys()), set([0,1,2]))
-        self.failUnlessEqual(b[0].remote_read(0, 25), "\x00" * 25)
+        self.failUnlessEqual(b[0].remote_read(0, 25), b"\x00" * 25)
 
     def test_advise_corruption(self):
         workdir = self.workdir("test_advise_corruption")
-        ss = StorageServer(workdir, "\x00" * 20, discard_storage=True)
+        ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
         si0_s = base32.b2a("si0")
@@ -782,7 +782,7 @@ class MutableServer(unittest.TestCase):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 
@@ -1489,7 +1489,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 
@@ -2873,7 +2873,7 @@ class Stats(unittest.TestCase):
 
     def create(self, name):
         workdir = self.workdir(name)
-        ss = StorageServer(workdir, "\x00" * 20)
+        ss = StorageServer(workdir, b"\x00" * 20)
         ss.setServiceParent(self.sparent)
         return ss
 

From 9ce43231b492a0e8f82f55b1a4d1b58bccf9067d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:49:04 -0400
Subject: [PATCH 02/13] More passing tests.

---
 src/allmydata/storage/server.py    |   2 +-
 src/allmydata/test/test_storage.py | 164 +++++++++++++++--------------
 2 files changed, 84 insertions(+), 82 deletions(-)

diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 3ffb58b68..387225b01 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -398,7 +398,7 @@ class StorageServer(service.MultiService, Referenceable):
         # since all shares get the same lease data, we just grab the leases
         # from the first share
         try:
-            shnum, filename = self._get_bucket_shares(storage_index).next()
+            shnum, filename = next(self._get_bucket_shares(storage_index))
             sf = ShareFile(filename)
             return sf.get_leases()
         except StopIteration:
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 9395ff8e7..bf0dda179 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,3 +1,5 @@
+from future.utils import native_str
+
 import time
 import os.path
 import platform
@@ -349,18 +351,18 @@ class Server(unittest.TestCase):
 
         ss = self.create("test_large_share")
 
-        already,writers = self.allocate(ss, "allocate", [0], 2**32+2)
+        already,writers = self.allocate(ss, b"allocate", [0], 2**32+2)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0]))
 
-        shnum, bucket = writers.items()[0]
+        shnum, bucket = list(writers.items())[0]
         # This test is going to hammer your filesystem if it doesn't make a sparse file for this.  :-(
-        bucket.remote_write(2**32, "ab")
+        bucket.remote_write(2**32, b"ab")
         bucket.remote_close()
 
-        readers = ss.remote_get_buckets("allocate")
+        readers = ss.remote_get_buckets(b"allocate")
         reader = readers[shnum]
-        self.failUnlessEqual(reader.remote_read(2**32, 2), "ab")
+        self.failUnlessEqual(reader.remote_read(2**32, 2), b"ab")
 
     def test_dont_overfill_dirs(self):
         """
@@ -369,9 +371,9 @@ class Server(unittest.TestCase):
         same storage index), this won't add an entry to the share directory.
         """
         ss = self.create("test_dont_overfill_dirs")
-        already, writers = self.allocate(ss, "storageindex", [0], 10)
+        already, writers = self.allocate(ss, b"storageindex", [0], 10)
         for i, wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         storedir = os.path.join(self.workdir("test_dont_overfill_dirs"),
                                 "shares")
@@ -379,9 +381,9 @@ class Server(unittest.TestCase):
 
         # Now store another one under another storageindex that has leading
         # chars the same as the first storageindex.
-        already, writers = self.allocate(ss, "storageindey", [0], 10)
+        already, writers = self.allocate(ss, b"storageindey", [0], 10)
         for i, wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         storedir = os.path.join(self.workdir("test_dont_overfill_dirs"),
                                 "shares")
@@ -392,7 +394,7 @@ class Server(unittest.TestCase):
         ss = self.create("test_remove_incoming")
         already, writers = self.allocate(ss, b"vid", range(3), 10)
         for i,wb in writers.items():
-            wb.remote_write(0, "%10d" % i)
+            wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
         incoming_share_dir = wb.incominghome
         incoming_bucket_dir = os.path.dirname(incoming_share_dir)
@@ -407,11 +409,11 @@ class Server(unittest.TestCase):
         # the allocated size of the bucket is not counted by the storage
         # server when accounting for space.
         ss = self.create("test_abort")
-        already, writers = self.allocate(ss, "allocate", [0, 1, 2], 150)
+        already, writers = self.allocate(ss, b"allocate", [0, 1, 2], 150)
         self.failIfEqual(ss.allocated_size(), 0)
 
         # Now abort the writers.
-        for writer in writers.itervalues():
+        for writer in writers.values():
             writer.remote_abort()
         self.failUnlessEqual(ss.allocated_size(), 0)
 
@@ -419,26 +421,26 @@ class Server(unittest.TestCase):
     def test_allocate(self):
         ss = self.create("test_allocate")
 
-        self.failUnlessEqual(ss.remote_get_buckets("allocate"), {})
+        self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {})
 
-        already,writers = self.allocate(ss, "allocate", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"allocate", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
 
         # while the buckets are open, they should not count as readable
-        self.failUnlessEqual(ss.remote_get_buckets("allocate"), {})
+        self.failUnlessEqual(ss.remote_get_buckets(b"allocate"), {})
 
         # close the buckets
         for i,wb in writers.items():
-            wb.remote_write(0, "%25d" % i)
+            wb.remote_write(0, b"%25d" % i)
             wb.remote_close()
             # aborting a bucket that was already closed is a no-op
             wb.remote_abort()
 
         # now they should be readable
-        b = ss.remote_get_buckets("allocate")
+        b = ss.remote_get_buckets(b"allocate")
         self.failUnlessEqual(set(b.keys()), set([0,1,2]))
-        self.failUnlessEqual(b[0].remote_read(0, 25), "%25d" % 0)
+        self.failUnlessEqual(b[0].remote_read(0, 25), b"%25d" % 0)
         b_str = str(b[0])
         self.failUnlessIn("BucketReader", b_str)
         self.failUnlessIn("mfwgy33dmf2g 0", b_str)
@@ -446,21 +448,21 @@ class Server(unittest.TestCase):
         # now if we ask about writing again, the server should offer those
         # three buckets as already present. It should offer them even if we
         # don't ask about those specific ones.
-        already,writers = self.allocate(ss, "allocate", [2,3,4], 75)
+        already,writers = self.allocate(ss, b"allocate", [2,3,4], 75)
         self.failUnlessEqual(already, set([0,1,2]))
         self.failUnlessEqual(set(writers.keys()), set([3,4]))
 
         # while those two buckets are open for writing, the server should
         # refuse to offer them to uploaders
 
-        already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75)
+        already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75)
         self.failUnlessEqual(already2, set([0,1,2]))
         self.failUnlessEqual(set(writers2.keys()), set([5]))
 
         # aborting the writes should remove the tempfiles
         for i,wb in writers2.items():
             wb.remote_abort()
-        already2,writers2 = self.allocate(ss, "allocate", [2,3,4,5], 75)
+        already2,writers2 = self.allocate(ss, b"allocate", [2,3,4,5], 75)
         self.failUnlessEqual(already2, set([0,1,2]))
         self.failUnlessEqual(set(writers2.keys()), set([5]))
 
@@ -471,27 +473,27 @@ class Server(unittest.TestCase):
 
     def test_bad_container_version(self):
         ss = self.create("test_bad_container_version")
-        a,w = self.allocate(ss, "si1", [0], 10)
-        w[0].remote_write(0, "\xff"*10)
+        a,w = self.allocate(ss, b"si1", [0], 10)
+        w[0].remote_write(0, b"\xff"*10)
         w[0].remote_close()
 
-        fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
+        fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
         f.write(struct.pack(">L", 0)) # this is invalid: minimum used is v1
         f.close()
 
-        ss.remote_get_buckets("allocate")
+        ss.remote_get_buckets(b"allocate")
 
         e = self.failUnlessRaises(UnknownImmutableContainerVersionError,
-                                  ss.remote_get_buckets, "si1")
+                                  ss.remote_get_buckets, b"si1")
         self.failUnlessIn(" had version 0 but we wanted 1", str(e))
 
     def test_disconnect(self):
         # simulate a disconnection
         ss = self.create("test_disconnect")
         canary = FakeCanary()
-        already,writers = self.allocate(ss, "disconnect", [0,1,2], 75, canary)
+        already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75, canary)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
         for (f,args,kwargs) in canary.disconnectors.values():
@@ -500,7 +502,7 @@ class Server(unittest.TestCase):
         del writers
 
         # that ought to delete the incoming shares
-        already,writers = self.allocate(ss, "disconnect", [0,1,2], 75)
+        already,writers = self.allocate(ss, b"disconnect", [0,1,2], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([0,1,2]))
 
@@ -549,7 +551,7 @@ class Server(unittest.TestCase):
         # become real, long-term allocation, and grows to include the
         # overhead.
         for bw in writers2.values():
-            bw.remote_write(0, "a"*25)
+            bw.remote_write(0, b"a"*25)
             bw.remote_close()
         del already2
         del writers2
@@ -561,7 +563,7 @@ class Server(unittest.TestCase):
 
         # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
         # 5000-1085=3915 free, therefore we can fit 39 100byte shares
-        already3, writers3 = self.allocate(ss,"vid3", range(100), 100, canary)
+        already3, writers3 = self.allocate(ss, b"vid3", range(100), 100, canary)
         self.failUnlessEqual(len(writers3), 39)
         self.failUnlessEqual(len(ss._active_writers), 39)
 
@@ -599,60 +601,60 @@ class Server(unittest.TestCase):
         sharenums = range(5)
         size = 100
 
-        rs0,cs0 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si0", rs0, cs0,
+        rs0,cs0 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si0", rs0, cs0,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 0)
         self.failUnlessEqual(len(writers), 5)
         for wb in writers.values():
             wb.remote_close()
 
-        leases = list(ss.get_leases("si0"))
+        leases = list(ss.get_leases(b"si0"))
         self.failUnlessEqual(len(leases), 1)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0]))
 
-        rs1,cs1 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si1", rs1, cs1,
+        rs1,cs1 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si1", rs1, cs1,
                                                      sharenums, size, canary)
         for wb in writers.values():
             wb.remote_close()
 
         # take out a second lease on si1
-        rs2,cs2 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si1", rs2, cs2,
+        rs2,cs2 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si1", rs2, cs2,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 5)
         self.failUnlessEqual(len(writers), 0)
 
-        leases = list(ss.get_leases("si1"))
+        leases = list(ss.get_leases(b"si1"))
         self.failUnlessEqual(len(leases), 2)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2]))
 
         # and a third lease, using add-lease
-        rs2a,cs2a = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                     hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        ss.remote_add_lease("si1", rs2a, cs2a)
-        leases = list(ss.get_leases("si1"))
+        rs2a,cs2a = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                     hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        ss.remote_add_lease(b"si1", rs2a, cs2a)
+        leases = list(ss.get_leases(b"si1"))
         self.failUnlessEqual(len(leases), 3)
         self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2, rs2a]))
 
         # add-lease on a missing storage index is silently ignored
-        self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None)
+        self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None)
 
         # check that si0 is readable
-        readers = ss.remote_get_buckets("si0")
+        readers = ss.remote_get_buckets(b"si0")
         self.failUnlessEqual(len(readers), 5)
 
         # renew the first lease. Only the proper renew_secret should work
-        ss.remote_renew_lease("si0", rs0)
-        self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", cs0)
-        self.failUnlessRaises(IndexError, ss.remote_renew_lease, "si0", rs1)
+        ss.remote_renew_lease(b"si0", rs0)
+        self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", cs0)
+        self.failUnlessRaises(IndexError, ss.remote_renew_lease, b"si0", rs1)
 
         # check that si0 is still readable
-        readers = ss.remote_get_buckets("si0")
+        readers = ss.remote_get_buckets(b"si0")
         self.failUnlessEqual(len(readers), 5)
 
         # There is no such method as remote_cancel_lease for now -- see
@@ -661,30 +663,30 @@ class Server(unittest.TestCase):
                         "ss should not have a 'remote_cancel_lease' method/attribute")
 
         # test overlapping uploads
-        rs3,cs3 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        rs4,cs4 = (hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()),
-                   hashutil.tagged_hash("blah", "%d" % self._lease_secret.next()))
-        already,writers = ss.remote_allocate_buckets("si3", rs3, cs3,
+        rs3,cs3 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        rs4,cs4 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
+                   hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)))
+        already,writers = ss.remote_allocate_buckets(b"si3", rs3, cs3,
                                                      sharenums, size, canary)
         self.failUnlessEqual(len(already), 0)
         self.failUnlessEqual(len(writers), 5)
-        already2,writers2 = ss.remote_allocate_buckets("si3", rs4, cs4,
+        already2,writers2 = ss.remote_allocate_buckets(b"si3", rs4, cs4,
                                                        sharenums, size, canary)
         self.failUnlessEqual(len(already2), 0)
         self.failUnlessEqual(len(writers2), 0)
         for wb in writers.values():
             wb.remote_close()
 
-        leases = list(ss.get_leases("si3"))
+        leases = list(ss.get_leases(b"si3"))
         self.failUnlessEqual(len(leases), 1)
 
-        already3,writers3 = ss.remote_allocate_buckets("si3", rs4, cs4,
+        already3,writers3 = ss.remote_allocate_buckets(b"si3", rs4, cs4,
                                                        sharenums, size, canary)
         self.failUnlessEqual(len(already3), 5)
         self.failUnlessEqual(len(writers3), 0)
 
-        leases = list(ss.get_leases("si3"))
+        leases = list(ss.get_leases(b"si3"))
         self.failUnlessEqual(len(leases), 2)
 
     def test_readonly(self):
@@ -727,44 +729,44 @@ class Server(unittest.TestCase):
         ss = StorageServer(workdir, b"\x00" * 20, discard_storage=True)
         ss.setServiceParent(self.sparent)
 
-        si0_s = base32.b2a("si0")
-        ss.remote_advise_corrupt_share("immutable", "si0", 0,
+        si0_s = base32.b2a(b"si0")
+        ss.remote_advise_corrupt_share(b"immutable", b"si0", 0,
                                        "This share smells funny.\n")
         reportdir = os.path.join(workdir, "corruption-advisories")
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 1)
         report_si0 = reports[0]
-        self.failUnlessIn(si0_s, report_si0)
-        f = open(os.path.join(reportdir, report_si0), "r")
+        self.failUnlessIn(native_str(si0_s), report_si0)
+        f = open(os.path.join(reportdir, report_si0), "rb")
         report = f.read()
         f.close()
-        self.failUnlessIn("type: immutable", report)
-        self.failUnlessIn("storage_index: %s" % si0_s, report)
-        self.failUnlessIn("share_number: 0", report)
-        self.failUnlessIn("This share smells funny.", report)
+        self.failUnlessIn(b"type: immutable", report)
+        self.failUnlessIn(b"storage_index: %s" % si0_s, report)
+        self.failUnlessIn(b"share_number: 0", report)
+        self.failUnlessIn(b"This share smells funny.", report)
 
         # test the RIBucketWriter version too
-        si1_s = base32.b2a("si1")
-        already,writers = self.allocate(ss, "si1", [1], 75)
+        si1_s = base32.b2a(b"si1")
+        already,writers = self.allocate(ss, b"si1", [1], 75)
         self.failUnlessEqual(already, set())
         self.failUnlessEqual(set(writers.keys()), set([1]))
-        writers[1].remote_write(0, "data")
+        writers[1].remote_write(0, b"data")
         writers[1].remote_close()
 
-        b = ss.remote_get_buckets("si1")
+        b = ss.remote_get_buckets(b"si1")
         self.failUnlessEqual(set(b.keys()), set([1]))
         b[1].remote_advise_corrupt_share("This share tastes like dust.\n")
 
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 2)
         report_si1 = [r for r in reports if si1_s in r][0]
-        f = open(os.path.join(reportdir, report_si1), "r")
+        f = open(os.path.join(reportdir, report_si1), "rb")
         report = f.read()
         f.close()
-        self.failUnlessIn("type: immutable", report)
-        self.failUnlessIn("storage_index: %s" % si1_s, report)
-        self.failUnlessIn("share_number: 1", report)
-        self.failUnlessIn("This share tastes like dust.", report)
+        self.failUnlessIn(b"type: immutable", report)
+        self.failUnlessIn(b"storage_index: %s" % si1_s, report)
+        self.failUnlessIn(b"share_number: 1", report)
+        self.failUnlessIn(b"This share tastes like dust.", report)
 
 
 
@@ -817,7 +819,7 @@ class MutableServer(unittest.TestCase):
 
     def test_bad_magic(self):
         ss = self.create("test_bad_magic")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(), set([0]), 10)
+        self.allocate(ss, "si1", "we1", next(self._lease_secret), set([0]), 10)
         fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
@@ -831,7 +833,7 @@ class MutableServer(unittest.TestCase):
 
     def test_container_size(self):
         ss = self.create("test_container_size")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         read = ss.remote_slot_readv
         rstaraw = ss.remote_slot_testv_and_readv_and_writev
@@ -929,7 +931,7 @@ class MutableServer(unittest.TestCase):
 
     def test_allocate(self):
         ss = self.create("test_allocate")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
 
         read = ss.remote_slot_readv
@@ -1315,7 +1317,7 @@ class MutableServer(unittest.TestCase):
 
     def test_remove(self):
         ss = self.create("test_remove")
-        self.allocate(ss, "si1", "we1", self._lease_secret.next(),
+        self.allocate(ss, "si1", "we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         readv = ss.remote_slot_readv
         writev = ss.remote_slot_testv_and_readv_and_writev

From 1a65dfa4abddaf758b43d428f2ce6f6a8d1d5e7b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 27 Aug 2020 15:58:03 -0400
Subject: [PATCH 03/13] Some potential progress.

---
 src/allmydata/test/test_storage.py | 68 ++++++++++++++++--------------
 1 file changed, 37 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index bf0dda179..647050b13 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -792,13 +792,19 @@ class MutableServer(unittest.TestCase):
         self.create("test_create")
 
     def write_enabler(self, we_tag):
-        return hashutil.tagged_hash("we_blah", we_tag)
+        return hashutil.tagged_hash(b"we_blah", we_tag)
 
     def renew_secret(self, tag):
-        return hashutil.tagged_hash("renew_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % (tag,)
+        assert isinstance(tag, bytes)
+        return hashutil.tagged_hash(b"renew_blah", tag)
 
     def cancel_secret(self, tag):
-        return hashutil.tagged_hash("cancel_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % (tag,)
+        assert isinstance(tag, bytes)
+        return hashutil.tagged_hash(b"cancel_blah", tag)
 
     def allocate(self, ss, storage_index, we_tag, lease_tag, sharenums, size):
         write_enabler = self.write_enabler(we_tag)
@@ -819,29 +825,29 @@ class MutableServer(unittest.TestCase):
 
     def test_bad_magic(self):
         ss = self.create("test_bad_magic")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret), set([0]), 10)
-        fn = os.path.join(ss.sharedir, storage_index_to_dir("si1"), "0")
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret), set([0]), 10)
+        fn = os.path.join(ss.sharedir, storage_index_to_dir(b"si1"), "0")
         f = open(fn, "rb+")
         f.seek(0)
-        f.write("BAD MAGIC")
+        f.write(b"BAD MAGIC")
         f.close()
         read = ss.remote_slot_readv
         e = self.failUnlessRaises(UnknownMutableContainerVersionError,
-                                  read, "si1", [0], [(0,10)])
+                                  read, b"si1", [0], [(0,10)])
         self.failUnlessIn(" had magic ", str(e))
         self.failUnlessIn(" but we wanted ", str(e))
 
     def test_container_size(self):
         ss = self.create("test_container_size")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         read = ss.remote_slot_readv
         rstaraw = ss.remote_slot_testv_and_readv_and_writev
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
-        answer = rstaraw("si1", secrets,
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], len(data)+12)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
@@ -850,33 +856,33 @@ class MutableServer(unittest.TestCase):
         # whose offset is too high) will raise an exception.
         TOOBIG = MutableShareFile.MAX_SIZE + 10
         self.failUnlessRaises(DataTooLargeError,
-                              rstaraw, "si1", secrets,
+                              rstaraw, b"si1", secrets,
                               {0: ([], [(TOOBIG,data)], None)},
                               [])
 
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], None)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {0: [data[:10]]})
 
         # Sending a new_length shorter than the current length truncates the
         # data.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 9)},
                          [])
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {0: [data[:9]]})
 
         # Sending a new_length longer than the current length doesn't change
         # the data.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 20)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
-        read_answer = read("si1", [0], [(0, 20)])
+        read_answer = read(b"si1", [0], [(0, 20)])
         self.failUnlessEqual(read_answer, {0: [data[:9]]})
 
         # Sending a write vector whose start is after the end of the current
@@ -884,35 +890,35 @@ class MutableServer(unittest.TestCase):
         # but instead fills with zeroes.
 
         # To test this, we fill the data area with a recognizable pattern.
-        pattern = ''.join([chr(i) for i in range(100)])
-        answer = rstaraw("si1", secrets,
+        pattern = u''.join([chr(i) for i in range(100)]).encode("utf-8")
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0, pattern)], None)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Then truncate the data...
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], 20)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Just confirm that you get an empty string if you try to read from
         # past the (new) endpoint now.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], None)},
                          [(20, 1980)])
-        self.failUnlessEqual(answer, (True, {0:[''],1:[''],2:['']}))
+        self.failUnlessEqual(answer, (True, {0:[b''],1:[b''],2:[b'']}))
 
         # Then the extend the file by writing a vector which starts out past
         # the end...
-        answer = rstaraw("si1", secrets,
-                         {0: ([], [(50, 'hellothere')], None)},
+        answer = rstaraw(b"si1", secrets,
+                         {0: ([], [(50, b'hellothere')], None)},
                          [])
         assert answer == (True, {0:[],1:[],2:[]})
         # Now if you read the stuff between 20 (where we earlier truncated)
         # and 50, it had better be all zeroes.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [], None)},
                          [(20, 30)])
-        self.failUnlessEqual(answer, (True, {0:['\x00'*30],1:[''],2:['']}))
+        self.failUnlessEqual(answer, (True, {0:[b'\x00'*30],1:[b''],2:[b'']}))
 
         # Also see if the server explicitly declares that it supports this
         # feature.
@@ -921,12 +927,12 @@ class MutableServer(unittest.TestCase):
         self.failUnless(storage_v1_ver.get("fills-holes-with-zero-bytes"))
 
         # If the size is dropped to zero the share is deleted.
-        answer = rstaraw("si1", secrets,
+        answer = rstaraw(b"si1", secrets,
                          {0: ([], [(0,data)], 0)},
                          [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        read_answer = read("si1", [0], [(0,10)])
+        read_answer = read(b"si1", [0], [(0,10)])
         self.failUnlessEqual(read_answer, {})
 
     def test_allocate(self):

From 5ad5b79cdd1e7834ce7de0a2d5f399c8e0282dd6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 10:53:52 -0400
Subject: [PATCH 04/13] More passing tests.

---
 src/allmydata/storage/immutable.py |   2 +-
 src/allmydata/storage/mutable.py   |  18 +-
 src/allmydata/test/test_storage.py | 352 +++++++++++++++--------------
 3 files changed, 187 insertions(+), 185 deletions(-)

diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py
index 4ec70e1e7..32ba1a739 100644
--- a/src/allmydata/storage/immutable.py
+++ b/src/allmydata/storage/immutable.py
@@ -85,7 +85,7 @@ class ShareFile(object):
         seekpos = self._data_offset+offset
         actuallength = max(0, min(length, self._lease_offset-seekpos))
         if actuallength == 0:
-            return ""
+            return b""
         with open(self.home, 'rb') as f:
             f.seek(seekpos)
             return f.read(actuallength)
diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py
index c108dfe32..a1eddf6cc 100644
--- a/src/allmydata/storage/mutable.py
+++ b/src/allmydata/storage/mutable.py
@@ -113,7 +113,7 @@ class MutableShareFile(object):
             # start beyond the end of the data return an empty string.
             length = max(0, data_length-offset)
         if length == 0:
-            return ""
+            return b""
         precondition(offset+length <= data_length)
         f.seek(self.DATA_OFFSET+offset)
         data = f.read(length)
@@ -421,18 +421,18 @@ class MutableShareFile(object):
                     # self._change_container_size() here.
 
 def testv_compare(a, op, b):
-    assert op in ("lt", "le", "eq", "ne", "ge", "gt")
-    if op == "lt":
+    assert op in (b"lt", b"le", b"eq", b"ne", b"ge", b"gt")
+    if op == b"lt":
         return a < b
-    if op == "le":
+    if op == b"le":
         return a <= b
-    if op == "eq":
+    if op == b"eq":
         return a == b
-    if op == "ne":
+    if op == b"ne":
         return a != b
-    if op == "ge":
+    if op == b"ge":
         return a >= b
-    if op == "gt":
+    if op == b"gt":
         return a > b
     # never reached
 
@@ -441,7 +441,7 @@ class EmptyShare(object):
     def check_testv(self, testv):
         test_good = True
         for (offset, length, operator, specimen) in testv:
-            data = ""
+            data = b""
             if not testv_compare(data, operator, specimen):
                 test_good = False
                 break
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 647050b13..639780846 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,4 @@
-from future.utils import native_str
+from future.utils import native_str, PY3
 
 import time
 import os.path
@@ -937,72 +937,72 @@ class MutableServer(unittest.TestCase):
 
     def test_allocate(self):
         ss = self.create("test_allocate")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
 
         read = ss.remote_slot_readv
-        self.failUnlessEqual(read("si1", [0], [(0, 10)]),
-                             {0: [""]})
-        self.failUnlessEqual(read("si1", [], [(0, 10)]),
-                             {0: [""], 1: [""], 2: [""]})
-        self.failUnlessEqual(read("si1", [0], [(100, 10)]),
-                             {0: [""]})
+        self.failUnlessEqual(read(b"si1", [0], [(0, 10)]),
+                             {0: [b""]})
+        self.failUnlessEqual(read(b"si1", [], [(0, 10)]),
+                             {0: [b""], 1: [b""], 2: [b""]})
+        self.failUnlessEqual(read(b"si1", [0], [(100, 10)]),
+                             {0: [b""]})
 
         # try writing to one
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
-        answer = write("si1", secrets,
+        answer = write(b"si1", secrets,
                        {0: ([], [(0,data)], None)},
                        [])
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
 
-        self.failUnlessEqual(read("si1", [0], [(0,20)]),
-                             {0: ["00000000001111111111"]})
-        self.failUnlessEqual(read("si1", [0], [(95,10)]),
-                             {0: ["99999"]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,20)]),
+                             {0: [b"00000000001111111111"]})
+        self.failUnlessEqual(read(b"si1", [0], [(95,10)]),
+                             {0: [b"99999"]})
         #self.failUnlessEqual(s0.remote_get_length(), 100)
 
-        bad_secrets = ("bad write enabler", secrets[1], secrets[2])
+        bad_secrets = (b"bad write enabler", secrets[1], secrets[2])
         f = self.failUnlessRaises(BadWriteEnablerError,
-                                  write, "si1", bad_secrets,
+                                  write, b"si1", bad_secrets,
                                   {}, [])
-        self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", f)
+        self.failUnlessIn("The write enabler was recorded by nodeid 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'.", str(f))
 
         # this testv should fail
-        answer = write("si1", secrets,
-                       {0: ([(0, 12, "eq", "444444444444"),
-                             (20, 5, "eq", "22222"),
+        answer = write(b"si1", secrets,
+                       {0: ([(0, 12, b"eq", b"444444444444"),
+                             (20, 5, b"eq", b"22222"),
                              ],
-                            [(0, "x"*100)],
+                            [(0, b"x"*100)],
                             None),
                         },
                        [(0,12), (20,5)],
                        )
         self.failUnlessEqual(answer, (False,
-                                      {0: ["000000000011", "22222"],
-                                       1: ["", ""],
-                                       2: ["", ""],
+                                      {0: [b"000000000011", b"22222"],
+                                       1: [b"", b""],
+                                       2: [b"", b""],
                                        }))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
 
         # as should this one
-        answer = write("si1", secrets,
-                       {0: ([(10, 5, "lt", "11111"),
+        answer = write(b"si1", secrets,
+                       {0: ([(10, 5, b"lt", b"11111"),
                              ],
-                            [(0, "x"*100)],
+                            [(0, b"x"*100)],
                             None),
                         },
                        [(10,5)],
                        )
         self.failUnlessEqual(answer, (False,
-                                      {0: ["11111"],
-                                       1: [""],
-                                       2: [""]},
+                                      {0: [b"11111"],
+                                       1: [b""],
+                                       2: [b""]},
                                       ))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
 
 
     def test_operators(self):
@@ -1010,201 +1010,201 @@ class MutableServer(unittest.TestCase):
         # test both fail+pass, reset data after each one.
         ss = self.create("test_operators")
 
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
 
         def reset():
-            write("si1", secrets,
+            write(b"si1", secrets,
                   {0: ([], [(0,data)], None)},
                   [])
 
         reset()
 
         #  lt
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11110"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
-        self.failUnlessEqual(read("si1", [], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(read(b"si1", [], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "lt", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  le
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11110"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "le", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  eq
-        answer = write("si1", secrets, {0: ([(10, 5, "eq", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "eq", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  ne
-        answer = write("si1", secrets, {0: ([(10, 5, "ne", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ne", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
         #  ge
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11110"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11111"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "ge", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11112"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
         #  gt
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11110"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11110"),
                                              ],
-                                            [(0, "y"*100)],
+                                            [(0, b"y"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (True, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: ["y"*100]})
+        self.failUnlessEqual(answer, (True, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11111"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11111"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
-        answer = write("si1", secrets, {0: ([(10, 5, "gt", "11112"),
+        answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
         # finally, test some operators against empty shares
-        answer = write("si1", secrets, {1: ([(10, 5, "eq", "11112"),
+        answer = write(b"si1", secrets, {1: ([(10, 5, b"eq", b"11112"),
                                              ],
-                                            [(0, "x"*100)],
+                                            [(0, b"x"*100)],
                                             None,
                                             )}, [(10,5)])
-        self.failUnlessEqual(answer, (False, {0: ["11111"]}))
-        self.failUnlessEqual(read("si1", [0], [(0,100)]), {0: [data]})
+        self.failUnlessEqual(answer, (False, {0: [b"11111"]}))
+        self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]})
         reset()
 
     def test_readv(self):
         ss = self.create("test_readv")
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
-        data = [("%d" % i) * 100 for i in range(3)]
-        rc = write("si1", secrets,
+        data = [(b"%d" % i) * 100 for i in range(3)]
+        rc = write(b"si1", secrets,
                    {0: ([], [(0,data[0])], None),
                     1: ([], [(0,data[1])], None),
                     2: ([], [(0,data[2])], None),
                     }, [])
         self.failUnlessEqual(rc, (True, {}))
 
-        answer = read("si1", [], [(0, 10)])
-        self.failUnlessEqual(answer, {0: ["0"*10],
-                                      1: ["1"*10],
-                                      2: ["2"*10]})
+        answer = read(b"si1", [], [(0, 10)])
+        self.failUnlessEqual(answer, {0: [b"0"*10],
+                                      1: [b"1"*10],
+                                      2: [b"2"*10]})
 
     def compare_leases_without_timestamps(self, leases_a, leases_b):
         self.failUnlessEqual(len(leases_a), len(leases_b))
@@ -1230,19 +1230,19 @@ class MutableServer(unittest.TestCase):
     def test_leases(self):
         ss = self.create("test_leases")
         def secrets(n):
-            return ( self.write_enabler("we1"),
-                     self.renew_secret("we1-%d" % n),
-                     self.cancel_secret("we1-%d" % n) )
-        data = "".join([ ("%d" % i) * 10 for i in range(10) ])
+            return ( self.write_enabler(b"we1"),
+                     self.renew_secret(b"we1-%d" % n),
+                     self.cancel_secret(b"we1-%d" % n) )
+        data = b"".join([ (b"%d" % i) * 10 for i in range(10) ])
         write = ss.remote_slot_testv_and_readv_and_writev
         read = ss.remote_slot_readv
-        rc = write("si1", secrets(0), {0: ([], [(0,data)], None)}, [])
+        rc = write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(rc, (True, {}))
 
         # create a random non-numeric file in the bucket directory, to
         # exercise the code that's supposed to ignore those.
         bucket_dir = os.path.join(self.workdir("test_leases"),
-                                  "shares", storage_index_to_dir("si1"))
+                                  "shares", storage_index_to_dir(b"si1"))
         f = open(os.path.join(bucket_dir, "ignore_me.txt"), "w")
         f.write("you ought to be ignoring me\n")
         f.close()
@@ -1251,45 +1251,45 @@ class MutableServer(unittest.TestCase):
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # add-lease on a missing storage index is silently ignored
-        self.failUnlessEqual(ss.remote_add_lease("si18", "", ""), None)
+        self.failUnlessEqual(ss.remote_add_lease(b"si18", b"", b""), None)
 
         # re-allocate the slots and use the same secrets, that should update
         # the lease
-        write("si1", secrets(0), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(0), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # renew it directly
-        ss.remote_renew_lease("si1", secrets(0)[1])
+        ss.remote_renew_lease(b"si1", secrets(0)[1])
         self.failUnlessEqual(len(list(s0.get_leases())), 1)
 
         # now allocate them with a bunch of different secrets, to trigger the
         # extended lease code. Use add_lease for one of them.
-        write("si1", secrets(1), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(1), {0: ([], [(0,data)], None)}, [])
         self.failUnlessEqual(len(list(s0.get_leases())), 2)
         secrets2 = secrets(2)
-        ss.remote_add_lease("si1", secrets2[1], secrets2[2])
+        ss.remote_add_lease(b"si1", secrets2[1], secrets2[2])
         self.failUnlessEqual(len(list(s0.get_leases())), 3)
-        write("si1", secrets(3), {0: ([], [(0,data)], None)}, [])
-        write("si1", secrets(4), {0: ([], [(0,data)], None)}, [])
-        write("si1", secrets(5), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(3), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(4), {0: ([], [(0,data)], None)}, [])
+        write(b"si1", secrets(5), {0: ([], [(0,data)], None)}, [])
 
         self.failUnlessEqual(len(list(s0.get_leases())), 6)
 
         all_leases = list(s0.get_leases())
         # and write enough data to expand the container, forcing the server
         # to move the leases
-        write("si1", secrets(0),
+        write(b"si1", secrets(0),
               {0: ([], [(0,data)], 200), },
               [])
 
         # read back the leases, make sure they're still intact.
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
-        ss.remote_renew_lease("si1", secrets(0)[1])
-        ss.remote_renew_lease("si1", secrets(1)[1])
-        ss.remote_renew_lease("si1", secrets(2)[1])
-        ss.remote_renew_lease("si1", secrets(3)[1])
-        ss.remote_renew_lease("si1", secrets(4)[1])
+        ss.remote_renew_lease(b"si1", secrets(0)[1])
+        ss.remote_renew_lease(b"si1", secrets(1)[1])
+        ss.remote_renew_lease(b"si1", secrets(2)[1])
+        ss.remote_renew_lease(b"si1", secrets(3)[1])
+        ss.remote_renew_lease(b"si1", secrets(4)[1])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
         # get a new copy of the leases, with the current timestamps. Reading
         # data and failing to renew/cancel leases should leave the timestamps
@@ -1300,7 +1300,7 @@ class MutableServer(unittest.TestCase):
         # examine the exception thus raised, make sure the old nodeid is
         # present, to provide for share migration
         e = self.failUnlessRaises(IndexError,
-                                  ss.remote_renew_lease, "si1",
+                                  ss.remote_renew_lease, b"si1",
                                   secrets(20)[1])
         e_s = str(e)
         self.failUnlessIn("Unable to renew non-existent lease", e_s)
@@ -1310,56 +1310,58 @@ class MutableServer(unittest.TestCase):
         self.compare_leases(all_leases, list(s0.get_leases()))
 
         # reading shares should not modify the timestamp
-        read("si1", [], [(0,200)])
+        read(b"si1", [], [(0,200)])
         self.compare_leases(all_leases, list(s0.get_leases()))
 
-        write("si1", secrets(0),
-              {0: ([], [(200, "make me bigger")], None)}, [])
+        write(b"si1", secrets(0),
+              {0: ([], [(200, b"make me bigger")], None)}, [])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
-        write("si1", secrets(0),
-              {0: ([], [(500, "make me really bigger")], None)}, [])
+        write(b"si1", secrets(0),
+              {0: ([], [(500, b"make me really bigger")], None)}, [])
         self.compare_leases_without_timestamps(all_leases, list(s0.get_leases()))
 
     def test_remove(self):
         ss = self.create("test_remove")
-        self.allocate(ss, "si1", "we1", next(self._lease_secret),
+        self.allocate(ss, b"si1", b"we1", next(self._lease_secret),
                       set([0,1,2]), 100)
         readv = ss.remote_slot_readv
         writev = ss.remote_slot_testv_and_readv_and_writev
-        secrets = ( self.write_enabler("we1"),
-                    self.renew_secret("we1"),
-                    self.cancel_secret("we1") )
+        secrets = ( self.write_enabler(b"we1"),
+                    self.renew_secret(b"we1"),
+                    self.cancel_secret(b"we1") )
         # delete sh0 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {0: ([], [], 0)},
                         [])
         # the answer should mention all the shares that existed before the
         # write
         self.failUnlessEqual(answer, (True, {0:[],1:[],2:[]}) )
         # but a new read should show only sh1 and sh2
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
-                             {1: [""], 2: [""]})
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
+                             {1: [b""], 2: [b""]})
 
         # delete sh1 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {1: ([], [], 0)},
                         [])
         self.failUnlessEqual(answer, (True, {1:[],2:[]}) )
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
-                             {2: [""]})
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
+                             {2: [b""]})
 
         # delete sh2 by setting its size to zero
-        answer = writev("si1", secrets,
+        answer = writev(b"si1", secrets,
                         {2: ([], [], 0)},
                         [])
         self.failUnlessEqual(answer, (True, {2:[]}) )
-        self.failUnlessEqual(readv("si1", [], [(0,10)]),
+        self.failUnlessEqual(readv(b"si1", [], [(0,10)]),
                              {})
         # and the bucket directory should now be gone
-        si = base32.b2a("si1")
+        si = base32.b2a(b"si1")
         # note: this is a detail of the storage server implementation, and
         # may change in the future
+        if PY3:
+            si = si.decode("utf-8")
         prefix = si[:2]
         prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix)
         bucketdir = os.path.join(prefixdir, si)
@@ -1373,7 +1375,7 @@ class MutableServer(unittest.TestCase):
         """
         ss = self.create("test_writev_without_renew_lease")
 
-        storage_index = "si2"
+        storage_index = b"si2"
         secrets = (
             self.write_enabler(storage_index),
             self.renew_secret(storage_index),
@@ -1400,7 +1402,7 @@ class MutableServer(unittest.TestCase):
         When ``get_slot_leases`` is called for a slot for which the server has no
         shares, it returns an empty iterable.
         """
-        ss = self.create(b"test_get_slot_leases_empty_slot")
+        ss = self.create("test_get_slot_leases_empty_slot")
         self.assertEqual(
             list(ss.get_slot_leases(b"si1")),
             [],
@@ -1413,7 +1415,7 @@ class MutableServer(unittest.TestCase):
         """
         ss = self.create("test_remove_non_present")
 
-        storage_index = "si1"
+        storage_index = b"si1"
         secrets = (
             self.write_enabler(storage_index),
             self.renew_secret(storage_index),
@@ -1590,7 +1592,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # and the verification key
         data += self.verification_key
         # Then we'll add in gibberish until we get to the right point.
-        nulls = "".join([" " for i in xrange(len(data), share_data_offset)])
+        nulls = b"".join([b" " for i in xrange(len(data), share_data_offset)])
         data += nulls
 
         # Then the share data
@@ -1614,7 +1616,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         data = self.build_test_mdmf_share(tail_segment, empty)
         # Finally, we write the whole thing to the storage server in one
         # pass.
-        testvs = [(0, 1, "eq", "")]
+        testvs = [(0, 1, b"eq", b"")]
         tws = {}
         tws[0] = (testvs, [(0, data)], None)
         readv = [(0, 1)]
@@ -1655,7 +1657,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                               sharedata_offset,
                               encprivkey_offset,
                               eof_offset)
-        final_share = "".join([prefix,
+        final_share = b"".join([prefix,
                            offsets,
                            self.verification_key,
                            self.signature,
@@ -1680,7 +1682,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # read them. This method writes one, which resembles but is not
         write = self.ss.remote_slot_testv_and_readv_and_writev
         share = self.build_test_sdmf_share(empty)
-        testvs = [(0, 1, "eq", "")]
+        testvs = [(0, 1, b"eq", b"")]
         tws = {}
         tws[0] = (testvs, [(0, share)], None)
         readv = []
@@ -1690,7 +1692,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # Check that every method equals what we expect it to.
         d = defer.succeed(None)
         def _check_block_and_salt(block_and_salt):
@@ -1762,19 +1764,19 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_different_tail_segment_size(self):
         self.write_test_share_to_server("si1", tail_segment=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_block_and_salt(5)
         def _check_tail_segment(results):
             block, salt = results
             self.failUnlessEqual(len(block), 1)
-            self.failUnlessEqual(block, "a")
+            self.failUnlessEqual(block, b"a")
         d.addCallback(_check_tail_segment)
         return d
 
 
     def test_get_block_with_invalid_segnum(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "test invalid segnum",
@@ -1785,7 +1787,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_encoding_parameters_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_encoding_parameters()
         def _check_encoding_parameters(args):
             (k, n, segment_size, datalen) = args
@@ -1799,7 +1801,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_seqnum_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_seqnum()
         d.addCallback(lambda seqnum:
             self.failUnlessEqual(seqnum, 0))
@@ -1808,7 +1810,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_root_hash_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_root_hash()
         d.addCallback(lambda root_hash:
             self.failUnlessEqual(root_hash, self.root_hash))
@@ -1817,7 +1819,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_get_checkstring_first(self):
         self.write_test_share_to_server("si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_checkstring()
         d.addCallback(lambda checkstring:
             self.failUnlessEqual(checkstring, self.checkstring))
@@ -1996,11 +1998,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def serialize_blockhashes(self, blockhashes):
-        return "".join(blockhashes)
+        return b"".join(blockhashes)
 
 
     def serialize_sharehashes(self, sharehashes):
-        ret = "".join([struct.pack(">H32s", i, sharehashes[i])
+        ret = b"".join([struct.pack(">H32s", i, sharehashes[i])
                         for i in sorted(sharehashes.keys())])
         return ret
 

From 50007ac868cd199538056bf820201c11b3a9347d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 12:41:19 -0400
Subject: [PATCH 05/13] More passing tests.

---
 src/allmydata/mutable/layout.py    |  73 +++++-----
 src/allmydata/test/test_storage.py | 208 +++++++++++++++--------------
 2 files changed, 143 insertions(+), 138 deletions(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index 20f2df3aa..5f459921a 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -1,3 +1,4 @@
+from past.utils import old_div
 
 import struct
 from allmydata.mutable.common import NeedMoreDataError, UnknownVersionError, \
@@ -180,11 +181,11 @@ def pack_offsets(verification_key_length, signature_length,
 def pack_share(prefix, verification_key, signature,
                share_hash_chain, block_hash_tree,
                share_data, encprivkey):
-    share_hash_chain_s = "".join([struct.pack(">H32s", i, share_hash_chain[i])
-                                  for i in sorted(share_hash_chain.keys())])
+    share_hash_chain_s = b"".join([struct.pack(">H32s", i, share_hash_chain[i])
+                                   for i in sorted(share_hash_chain.keys())])
     for h in block_hash_tree:
         assert len(h) == 32
-    block_hash_tree_s = "".join(block_hash_tree)
+    block_hash_tree_s = b"".join(block_hash_tree)
 
     offsets = pack_offsets(len(verification_key),
                            len(signature),
@@ -192,14 +193,14 @@ def pack_share(prefix, verification_key, signature,
                            len(block_hash_tree_s),
                            len(share_data),
                            len(encprivkey))
-    final_share = "".join([prefix,
-                           offsets,
-                           verification_key,
-                           signature,
-                           share_hash_chain_s,
-                           block_hash_tree_s,
-                           share_data,
-                           encprivkey])
+    final_share = b"".join([prefix,
+                            offsets,
+                            verification_key,
+                            signature,
+                            share_hash_chain_s,
+                            block_hash_tree_s,
+                            share_data,
+                            encprivkey])
     return final_share
 
 def pack_prefix(seqnum, root_hash, IV,
@@ -255,7 +256,7 @@ class SDMFSlotWriteProxy(object):
                                                        self._required_shares)
         assert expected_segment_size == segment_size
 
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
 
         # This is meant to mimic how SDMF files were built before MDMF
         # entered the picture: we generate each share in its entirety,
@@ -343,7 +344,7 @@ class SDMFSlotWriteProxy(object):
             assert len(h) == HASH_SIZE
 
         # serialize the blockhashes, then set them.
-        blockhashes_s = "".join(blockhashes)
+        blockhashes_s = b"".join(blockhashes)
         self._share_pieces['block_hash_tree'] = blockhashes_s
 
         return defer.succeed(None)
@@ -354,12 +355,12 @@ class SDMFSlotWriteProxy(object):
         Add the share hash chain to the share.
         """
         assert isinstance(sharehashes, dict)
-        for h in sharehashes.itervalues():
+        for h in sharehashes.values():
             assert len(h) == HASH_SIZE
 
         # serialize the sharehashes, then set them.
-        sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
-                                 for i in sorted(sharehashes.keys())])
+        sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
+                                  for i in sorted(sharehashes.keys())])
         self._share_pieces['share_hash_chain'] = sharehashes_s
 
         return defer.succeed(None)
@@ -383,7 +384,7 @@ class SDMFSlotWriteProxy(object):
         assert len(salt) == SALT_SIZE
 
         self._share_pieces['salt'] = salt
-        self._share_pieces['sharedata'] = ""
+        self._share_pieces['sharedata'] = b""
 
 
     def get_signable(self):
@@ -519,14 +520,14 @@ class SDMFSlotWriteProxy(object):
         # to the remote server in one write.
         offsets = self._pack_offsets()
         prefix = self.get_signable()
-        final_share = "".join([prefix,
-                               offsets,
-                               self._share_pieces['verification_key'],
-                               self._share_pieces['signature'],
-                               self._share_pieces['share_hash_chain'],
-                               self._share_pieces['block_hash_tree'],
-                               self._share_pieces['sharedata'],
-                               self._share_pieces['encprivkey']])
+        final_share = b"".join([prefix,
+                                offsets,
+                                self._share_pieces['verification_key'],
+                                self._share_pieces['signature'],
+                                self._share_pieces['share_hash_chain'],
+                                self._share_pieces['block_hash_tree'],
+                                self._share_pieces['sharedata'],
+                                self._share_pieces['encprivkey']])
 
         # Our only data vector is going to be writing the final share,
         # in its entirely.
@@ -788,7 +789,7 @@ class MDMFSlotWriteProxy(object):
         # and also because it provides a useful amount of bounds checking.
         self._num_segments = mathutil.div_ceil(self._data_length,
                                                self._segment_size)
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
         # We also calculate the share size, to help us with block
         # constraints later.
         tail_size = self._data_length % self._segment_size
@@ -797,7 +798,7 @@ class MDMFSlotWriteProxy(object):
         else:
             self._tail_block_size = mathutil.next_multiple(tail_size,
                                                            self._required_shares)
-            self._tail_block_size /= self._required_shares
+            self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
 
         # We already know where the sharedata starts; right after the end
         # of the header (which is defined as the signable part + the offsets)
@@ -868,7 +869,7 @@ class MDMFSlotWriteProxy(object):
         else:
             checkstring = seqnum_or_checkstring
 
-        if checkstring == "":
+        if checkstring == b"":
             # We special-case this, since len("") = 0, but we need
             # length of 1 for the case of an empty share to work on the
             # storage server, which is what a checkstring that is the
@@ -876,7 +877,7 @@ class MDMFSlotWriteProxy(object):
             self._testvs = []
         else:
             self._testvs = []
-            self._testvs.append((0, len(checkstring), "eq", checkstring))
+            self._testvs.append((0, len(checkstring), b"eq", checkstring))
 
 
     def __repr__(self):
@@ -893,7 +894,7 @@ class MDMFSlotWriteProxy(object):
         if self._root_hash:
             roothash = self._root_hash
         else:
-            roothash = "\x00" * 32
+            roothash = b"\x00" * 32
         return struct.pack(MDMFCHECKSTRING,
                            1,
                            self._seqnum,
@@ -964,7 +965,7 @@ class MDMFSlotWriteProxy(object):
 
         assert isinstance(blockhashes, list)
 
-        blockhashes_s = "".join(blockhashes)
+        blockhashes_s = b"".join(blockhashes)
         self._offsets['EOF'] = self._offsets['block_hash_tree'] + len(blockhashes_s)
 
         self._writevs.append(tuple([self._offsets['block_hash_tree'],
@@ -998,7 +999,7 @@ class MDMFSlotWriteProxy(object):
         if "verification_key" in self._offsets:
             raise LayoutInvalid("You must write the share hash chain "
                                 "before you write the signature")
-        sharehashes_s = "".join([struct.pack(">H32s", i, sharehashes[i])
+        sharehashes_s = b"".join([struct.pack(">H32s", i, sharehashes[i])
                                   for i in sorted(sharehashes.keys())])
         self._offsets['signature'] = self._offsets['share_hash_chain'] + \
             len(sharehashes_s)
@@ -1149,7 +1150,7 @@ class MDMFSlotWriteProxy(object):
         tw_vectors = {}
         if not self._testvs:
             self._testvs = []
-            self._testvs.append(tuple([0, 1, "eq", ""]))
+            self._testvs.append(tuple([0, 1, b"eq", b""]))
         if not self._written:
             # Write a new checkstring to the share when we write it, so
             # that we have something to check later.
@@ -1157,7 +1158,7 @@ class MDMFSlotWriteProxy(object):
             datavs.append((0, new_checkstring))
             def _first_write():
                 self._written = True
-                self._testvs = [(0, len(new_checkstring), "eq", new_checkstring)]
+                self._testvs = [(0, len(new_checkstring), b"eq", new_checkstring)]
             on_success = _first_write
         tw_vectors[self.shnum] = (self._testvs, datavs, None)
         d = self._storage_server.slot_testv_and_readv_and_writev(
@@ -1317,7 +1318,7 @@ class MDMFSlotReadProxy(object):
         self._segment_size = segsize
         self._data_length = datalen
 
-        self._block_size = self._segment_size / self._required_shares
+        self._block_size = old_div(self._segment_size, self._required_shares)
         # We can upload empty files, and need to account for this fact
         # so as to avoid zero-division and zero-modulo errors.
         if datalen > 0:
@@ -1329,7 +1330,7 @@ class MDMFSlotReadProxy(object):
         else:
             self._tail_block_size = mathutil.next_multiple(tail_size,
                                                     self._required_shares)
-            self._tail_block_size /= self._required_shares
+            self._tail_block_size = old_div(self._tail_block_size, self._required_shares)
 
         return encoding_parameters
 
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 639780846..6208af0fa 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1451,22 +1451,22 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         self.ss = self.create("MDMFProxies storage test server")
         self.rref = RemoteBucket(self.ss)
         self.storage_server = _StorageServer(lambda: self.rref)
-        self.secrets = (self.write_enabler("we_secret"),
-                        self.renew_secret("renew_secret"),
-                        self.cancel_secret("cancel_secret"))
-        self.segment = "aaaaaa"
-        self.block = "aa"
-        self.salt = "a" * 16
-        self.block_hash = "a" * 32
-        self.block_hash_tree = [self.block_hash for i in xrange(6)]
+        self.secrets = (self.write_enabler(b"we_secret"),
+                        self.renew_secret(b"renew_secret"),
+                        self.cancel_secret(b"cancel_secret"))
+        self.segment = b"aaaaaa"
+        self.block = b"aa"
+        self.salt = b"a" * 16
+        self.block_hash = b"a" * 32
+        self.block_hash_tree = [self.block_hash for i in range(6)]
         self.share_hash = self.block_hash
-        self.share_hash_chain = dict([(i, self.share_hash) for i in xrange(6)])
-        self.signature = "foobarbaz"
-        self.verification_key = "vvvvvv"
-        self.encprivkey = "private"
+        self.share_hash_chain = dict([(i, self.share_hash) for i in range(6)])
+        self.signature = b"foobarbaz"
+        self.verification_key = b"vvvvvv"
+        self.encprivkey = b"private"
         self.root_hash = self.block_hash
         self.salt_hash = self.root_hash
-        self.salt_hash_tree = [self.salt_hash for i in xrange(6)]
+        self.salt_hash_tree = [self.salt_hash for i in range(6)]
         self.block_hash_tree_s = self.serialize_blockhashes(self.block_hash_tree)
         self.share_hash_chain_s = self.serialize_sharehashes(self.share_hash_chain)
         # blockhashes and salt hashes are serialized in the same way,
@@ -1481,15 +1481,19 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def write_enabler(self, we_tag):
-        return hashutil.tagged_hash("we_blah", we_tag)
+        return hashutil.tagged_hash(b"we_blah", we_tag)
 
 
     def renew_secret(self, tag):
-        return hashutil.tagged_hash("renew_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % tag
+        return hashutil.tagged_hash(b"renew_blah", tag)
 
 
     def cancel_secret(self, tag):
-        return hashutil.tagged_hash("cancel_blah", str(tag))
+        if isinstance(tag, int):
+            tag = b"%d" % tag
+        return hashutil.tagged_hash(b"cancel_blah", tag)
 
 
     def workdir(self, name):
@@ -1531,14 +1535,14 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                                 6,
                                 36)
         # Now we'll build the offsets.
-        sharedata = ""
+        sharedata = b""
         if not tail_segment and not empty:
-            for i in xrange(6):
+            for i in range(6):
                 sharedata += self.salt + self.block
         elif tail_segment:
-            for i in xrange(5):
+            for i in range(5):
                 sharedata += self.salt + self.block
-            sharedata += self.salt + "a"
+            sharedata += self.salt + b"a"
 
         # The encrypted private key comes after the shares + salts
         offset_size = struct.calcsize(MDMFOFFSETS)
@@ -1592,7 +1596,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # and the verification key
         data += self.verification_key
         # Then we'll add in gibberish until we get to the right point.
-        nulls = b"".join([b" " for i in xrange(len(data), share_data_offset)])
+        nulls = b"".join([b" " for i in range(len(data), share_data_offset)])
         data += nulls
 
         # Then the share data
@@ -1626,11 +1630,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def build_test_sdmf_share(self, empty=False):
         if empty:
-            sharedata = ""
+            sharedata = b""
         else:
             sharedata = self.segment * 6
         self.sharedata = sharedata
-        blocksize = len(sharedata) / 3
+        blocksize = len(sharedata) // 3
         block = sharedata[:blocksize]
         self.blockdata = block
         prefix = struct.pack(">BQ32s16s BBQQ",
@@ -1691,7 +1695,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # Check that every method equals what we expect it to.
         d = defer.succeed(None)
@@ -1700,7 +1704,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnlessEqual(block, self.block)
             self.failUnlessEqual(salt, self.salt)
 
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mr.get_block_and_salt(i))
             d.addCallback(_check_block_and_salt)
@@ -1763,7 +1767,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read_with_different_tail_segment_size(self):
-        self.write_test_share_to_server("si1", tail_segment=True)
+        self.write_test_share_to_server(b"si1", tail_segment=True)
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_block_and_salt(5)
         def _check_tail_segment(results):
@@ -1775,7 +1779,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_block_with_invalid_segnum(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -1786,7 +1790,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_encoding_parameters_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_encoding_parameters()
         def _check_encoding_parameters(args):
@@ -1800,7 +1804,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_seqnum_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_seqnum()
         d.addCallback(lambda seqnum:
@@ -1809,7 +1813,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_root_hash_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_root_hash()
         d.addCallback(lambda root_hash:
@@ -1818,7 +1822,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_get_checkstring_first(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.get_checkstring()
         d.addCallback(lambda checkstring:
@@ -1832,9 +1836,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # the test vectors failed, this read vector can help us to
         # diagnose the problem. This test ensures that the read vector
         # is working appropriately.
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
 
-        for i in xrange(6):
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
         mw.put_encprivkey(self.encprivkey)
         mw.put_blockhashes(self.block_hash_tree)
@@ -1849,7 +1853,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnless(result)
             self.failIf(readv)
             self.old_checkstring = mw.get_checkstring()
-            mw.set_checkstring("")
+            mw.set_checkstring(b"")
         d.addCallback(_then)
         d.addCallback(lambda ignored:
             mw.finish_publishing())
@@ -1866,9 +1870,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_private_key_after_share_hash_chain(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -1885,10 +1889,10 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_signature_after_verification_key(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         d = defer.succeed(None)
         # Put everything up to and including the verification key.
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -1915,8 +1919,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # Make two mutable writers, both pointing to the same storage
         # server, both at the same storage index, and try writing to the
         # same share.
-        mw1 = self._make_new_mw("si1", 0)
-        mw2 = self._make_new_mw("si1", 0)
+        mw1 = self._make_new_mw(b"si1", 0)
+        mw2 = self._make_new_mw(b"si1", 0)
 
         def _check_success(results):
             result, readvs = results
@@ -1927,7 +1931,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failIf(result)
 
         def _write_share(mw):
-            for i in xrange(6):
+            for i in range(6):
                 mw.put_block(self.block, i, self.salt)
             mw.put_encprivkey(self.encprivkey)
             mw.put_blockhashes(self.block_hash_tree)
@@ -1947,9 +1951,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_invalid_salt_size(self):
         # Salts need to be 16 bytes in size. Writes that attempt to
         # write more or less than this should be rejected.
-        mw = self._make_new_mw("si1", 0)
-        invalid_salt = "a" * 17 # 17 bytes
-        another_invalid_salt = "b" * 15 # 15 bytes
+        mw = self._make_new_mw(b"si1", 0)
+        invalid_salt = b"a" * 17 # 17 bytes
+        another_invalid_salt = b"b" * 15 # 15 bytes
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "salt too big",
@@ -1977,9 +1981,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             res, d = results
             self.failUnless(results)
 
-        mw = self._make_new_mw("si1", 0)
-        mw.set_checkstring("this is a lie")
-        for i in xrange(6):
+        mw = self._make_new_mw(b"si1", 0)
+        mw.set_checkstring(b"this is a lie")
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
         mw.put_encprivkey(self.encprivkey)
         mw.put_blockhashes(self.block_hash_tree)
@@ -1990,7 +1994,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         d = mw.finish_publishing()
         d.addCallback(_check_failure)
         d.addCallback(lambda ignored:
-            mw.set_checkstring(""))
+            mw.set_checkstring(b""))
         d.addCallback(lambda ignored:
             mw.finish_publishing())
         d.addCallback(_check_success)
@@ -2010,7 +2014,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write(self):
         # This translates to a file with 6 6-byte segments, and with 2-byte
         # blocks.
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         # Test writing some blocks.
         read = self.ss.remote_slot_readv
         expected_private_key_offset = struct.calcsize(MDMFHEADER)
@@ -2021,7 +2025,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                                     SHARE_HASH_CHAIN_SIZE
         written_block_size = 2 + len(self.salt)
         written_block = self.block + self.salt
-        for i in xrange(6):
+        for i in range(6):
             mw.put_block(self.block, i, self.salt)
 
         mw.put_encprivkey(self.encprivkey)
@@ -2035,35 +2039,35 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.failUnlessEqual(len(results), 2)
             result, ign = results
             self.failUnless(result, "publish failed")
-            for i in xrange(6):
-                self.failUnlessEqual(read("si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]),
+            for i in range(6):
+                self.failUnlessEqual(read(b"si1", [0], [(expected_sharedata_offset + (i * written_block_size), written_block_size)]),
                                 {0: [written_block]})
 
             self.failUnlessEqual(len(self.encprivkey), 7)
-            self.failUnlessEqual(read("si1", [0], [(expected_private_key_offset, 7)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_private_key_offset, 7)]),
                                  {0: [self.encprivkey]})
 
             expected_block_hash_offset = expected_sharedata_offset + \
                         (6 * written_block_size)
             self.failUnlessEqual(len(self.block_hash_tree_s), 32 * 6)
-            self.failUnlessEqual(read("si1", [0], [(expected_block_hash_offset, 32 * 6)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_block_hash_offset, 32 * 6)]),
                                  {0: [self.block_hash_tree_s]})
 
             expected_share_hash_offset = expected_private_key_offset + len(self.encprivkey)
-            self.failUnlessEqual(read("si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]),
+            self.failUnlessEqual(read(b"si1", [0],[(expected_share_hash_offset, (32 + 2) * 6)]),
                                  {0: [self.share_hash_chain_s]})
 
-            self.failUnlessEqual(read("si1", [0], [(9, 32)]),
+            self.failUnlessEqual(read(b"si1", [0], [(9, 32)]),
                                  {0: [self.root_hash]})
             expected_signature_offset = expected_share_hash_offset + \
                 len(self.share_hash_chain_s)
             self.failUnlessEqual(len(self.signature), 9)
-            self.failUnlessEqual(read("si1", [0], [(expected_signature_offset, 9)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_signature_offset, 9)]),
                                  {0: [self.signature]})
 
             expected_verification_key_offset = expected_signature_offset + len(self.signature)
             self.failUnlessEqual(len(self.verification_key), 6)
-            self.failUnlessEqual(read("si1", [0], [(expected_verification_key_offset, 6)]),
+            self.failUnlessEqual(read(b"si1", [0], [(expected_verification_key_offset, 6)]),
                                  {0: [self.verification_key]})
 
             signable = mw.get_signable()
@@ -2082,49 +2086,49 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
             # Check the version number to make sure that it is correct.
             expected_version_number = struct.pack(">B", 1)
-            self.failUnlessEqual(read("si1", [0], [(0, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(0, 1)]),
                                  {0: [expected_version_number]})
             # Check the sequence number to make sure that it is correct
             expected_sequence_number = struct.pack(">Q", 0)
-            self.failUnlessEqual(read("si1", [0], [(1, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(1, 8)]),
                                  {0: [expected_sequence_number]})
             # Check that the encoding parameters (k, N, segement size, data
             # length) are what they should be. These are  3, 10, 6, 36
             expected_k = struct.pack(">B", 3)
-            self.failUnlessEqual(read("si1", [0], [(41, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(41, 1)]),
                                  {0: [expected_k]})
             expected_n = struct.pack(">B", 10)
-            self.failUnlessEqual(read("si1", [0], [(42, 1)]),
+            self.failUnlessEqual(read(b"si1", [0], [(42, 1)]),
                                  {0: [expected_n]})
             expected_segment_size = struct.pack(">Q", 6)
-            self.failUnlessEqual(read("si1", [0], [(43, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(43, 8)]),
                                  {0: [expected_segment_size]})
             expected_data_length = struct.pack(">Q", 36)
-            self.failUnlessEqual(read("si1", [0], [(51, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(51, 8)]),
                                  {0: [expected_data_length]})
             expected_offset = struct.pack(">Q", expected_private_key_offset)
-            self.failUnlessEqual(read("si1", [0], [(59, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(59, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_share_hash_offset)
-            self.failUnlessEqual(read("si1", [0], [(67, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(67, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_signature_offset)
-            self.failUnlessEqual(read("si1", [0], [(75, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(75, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_verification_key_offset)
-            self.failUnlessEqual(read("si1", [0], [(83, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(83, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_verification_key_offset + len(self.verification_key))
-            self.failUnlessEqual(read("si1", [0], [(91, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(91, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_sharedata_offset)
-            self.failUnlessEqual(read("si1", [0], [(99, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(99, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_block_hash_offset)
-            self.failUnlessEqual(read("si1", [0], [(107, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(107, 8)]),
                                  {0: [expected_offset]})
             expected_offset = struct.pack(">Q", expected_eof_offset)
-            self.failUnlessEqual(read("si1", [0], [(115, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(115, 8)]),
                                  {0: [expected_offset]})
         d.addCallback(_check_publish)
         return d
@@ -2140,13 +2144,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_write_rejected_with_too_many_blocks(self):
-        mw = self._make_new_mw("si0", 0)
+        mw = self._make_new_mw(b"si0", 0)
 
         # Try writing too many blocks. We should not be able to write
         # more than 6
         # blocks into each share.
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2159,8 +2163,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write_rejected_with_invalid_salt(self):
         # Try writing an invalid salt. Salts are 16 bytes -- any more or
         # less should cause an error.
-        mw = self._make_new_mw("si1", 0)
-        bad_salt = "a" * 17 # 17 bytes
+        mw = self._make_new_mw(b"si1", 0)
+        bad_salt = b"a" * 17 # 17 bytes
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             self.shouldFail(LayoutInvalid, "test_invalid_salt",
@@ -2171,15 +2175,15 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_write_rejected_with_invalid_root_hash(self):
         # Try writing an invalid root hash. This should be SHA256d, and
         # 32 bytes long as a result.
-        mw = self._make_new_mw("si2", 0)
+        mw = self._make_new_mw(b"si2", 0)
         # 17 bytes != 32 bytes
-        invalid_root_hash = "a" * 17
+        invalid_root_hash = b"a" * 17
         d = defer.succeed(None)
         # Before this test can work, we need to put some blocks + salts,
         # a block hash tree, and a share hash tree. Otherwise, we'll see
         # failures that match what we are looking for, but are caused by
         # the constraints imposed on operation ordering.
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2199,8 +2203,8 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # _make_new_mw is 2bytes -- any more or any less than this
         # should be cause for failure, unless it is the tail segment, in
         # which case it may not be failure.
-        invalid_block = "a"
-        mw = self._make_new_mw("si3", 0, 33) # implies a tail segment with
+        invalid_block = b"a"
+        mw = self._make_new_mw(b"si3", 0, 33) # implies a tail segment with
                                              # one byte blocks
         # 1 bytes != 2 bytes
         d = defer.succeed(None)
@@ -2214,7 +2218,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.shouldFail(LayoutInvalid, "test blocksize too large",
                             None,
                             mw.put_block, invalid_block, 0, self.salt))
-        for i in xrange(5):
+        for i in range(5):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         # Try to put an invalid tail segment
@@ -2222,7 +2226,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
             self.shouldFail(LayoutInvalid, "test invalid tail segment",
                             None,
                             mw.put_block, self.block, 5, self.salt))
-        valid_block = "a"
+        valid_block = b"a"
         d.addCallback(lambda ignored:
             mw.put_block(valid_block, 5, self.salt))
         return d
@@ -2247,10 +2251,10 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         #  - share hashes and block hashes before root hash
         #  - root hash before signature
         #  - signature before verification key
-        mw0 = self._make_new_mw("si0", 0)
+        mw0 = self._make_new_mw(b"si0", 0)
         # Write some shares
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw0.put_block(self.block, i, self.salt))
 
@@ -2315,11 +2319,11 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_end_to_end(self):
-        mw = self._make_new_mw("si1", 0)
+        mw = self._make_new_mw(b"si1", 0)
         # Write a share using the mutable writer, and make sure that the
         # reader knows how to read everything back to us.
         d = defer.succeed(None)
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mw.put_block(self.block, i, self.salt))
         d.addCallback(lambda ignored:
@@ -2337,13 +2341,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         d.addCallback(lambda ignored:
             mw.finish_publishing())
 
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         def _check_block_and_salt(block_and_salt):
             (block, salt) = block_and_salt
             self.failUnlessEqual(block, self.block)
             self.failUnlessEqual(salt, self.salt)
 
-        for i in xrange(6):
+        for i in range(6):
             d.addCallback(lambda ignored, i=i:
                 mr.get_block_and_salt(i))
             d.addCallback(_check_block_and_salt)
@@ -2404,7 +2408,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # The MDMFSlotReadProxy should also know how to read SDMF files,
         # since it will encounter them on the grid. Callers use the
         # is_sdmf method to test this.
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = mr.is_sdmf()
         d.addCallback(lambda issdmf:
@@ -2415,7 +2419,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
     def test_reads_sdmf(self):
         # The slot read proxy should, naturally, know how to tell us
         # about data in the SDMF format
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2486,7 +2490,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # SDMF shares have only one segment, so it doesn't make sense to
         # read more segments than that. The reader should know this and
         # complain if we try to do that.
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2507,7 +2511,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # finding out which shares are on the remote peer so that it
         # doesn't waste round trips.
         mdmf_data = self.build_test_mdmf_share()
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         def _make_mr(ignored, length):
             mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, mdmf_data[:length])
             return mr
@@ -2568,7 +2572,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_prefetched_sdmf_data(self):
         sdmf_data = self.build_test_sdmf_share()
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         def _make_mr(ignored, length):
             mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, sdmf_data[:length])
             return mr
@@ -2635,7 +2639,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # Some tests upload a file with no contents to test things
         # unrelated to the actual handling of the content of the file.
         # The reader should behave intelligently in these cases.
-        self.write_test_share_to_server("si1", empty=True)
+        self.write_test_share_to_server(b"si1", empty=True)
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct.
@@ -2661,7 +2665,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_read_with_empty_sdmf_file(self):
-        self.write_sdmf_share_to_server("si1", empty=True)
+        self.write_sdmf_share_to_server(b"si1", empty=True)
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct
@@ -2687,7 +2691,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_verinfo_with_sdmf_file(self):
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         # We should be able to get the version information.
         d = defer.succeed(None)
@@ -2728,7 +2732,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
 
     def test_verinfo_with_mdmf_file(self):
-        self.write_test_share_to_server("si1")
+        self.write_test_share_to_server(b"si1")
         mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2804,7 +2808,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         def _then(ignored):
             self.failUnlessEqual(self.rref.write_count, 1)
             read = self.ss.remote_slot_readv
-            self.failUnlessEqual(read("si1", [0], [(0, len(data))]),
+            self.failUnlessEqual(read(b"si1", [0], [(0, len(data))]),
                                  {0: [data]})
         d.addCallback(_then)
         return d
@@ -2812,7 +2816,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_sdmf_writer_preexisting_share(self):
         data = self.build_test_sdmf_share()
-        self.write_sdmf_share_to_server("si1")
+        self.write_sdmf_share_to_server(b"si1")
 
         # Now there is a share on the storage server. To successfully
         # write, we need to set the checkstring correctly. When we
@@ -2861,9 +2865,9 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         def _then_again(results):
             self.failUnless(results[0])
             read = self.ss.remote_slot_readv
-            self.failUnlessEqual(read("si1", [0], [(1, 8)]),
+            self.failUnlessEqual(read(b"si1", [0], [(1, 8)]),
                                  {0: [struct.pack(">Q", 1)]})
-            self.failUnlessEqual(read("si1", [0], [(9, len(data) - 9)]),
+            self.failUnlessEqual(read(b"si1", [0], [(9, len(data) - 9)]),
                                  {0: [data[9:]]})
         d.addCallback(_then_again)
         return d

From 2ba0854e0d80abedb1e5e9d8fc9fc6fc1af746cf Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 28 Aug 2020 12:59:03 -0400
Subject: [PATCH 06/13] More passing tests.

---
 src/allmydata/mutable/layout.py    | 14 +++++++-------
 src/allmydata/test/test_storage.py | 24 ++++++++++++------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index 5f459921a..ee086ec5e 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -297,7 +297,7 @@ class SDMFSlotWriteProxy(object):
                                       salt)
         else:
             checkstring = checkstring_or_seqnum
-        self._testvs = [(0, len(checkstring), "eq", checkstring)]
+        self._testvs = [(0, len(checkstring), b"eq", checkstring)]
 
 
     def get_checkstring(self):
@@ -307,7 +307,7 @@ class SDMFSlotWriteProxy(object):
         """
         if self._testvs:
             return self._testvs[0][3]
-        return ""
+        return b""
 
 
     def put_block(self, data, segnum, salt):
@@ -538,7 +538,7 @@ class SDMFSlotWriteProxy(object):
             # yet, so we assume that we are writing a new share, and set
             # a test vector that will allow a new share to be written.
             self._testvs = []
-            self._testvs.append(tuple([0, 1, "eq", ""]))
+            self._testvs.append(tuple([0, 1, b"eq", b""]))
 
         tw_vectors = {}
         tw_vectors[self.shnum] = (self._testvs, datavs, None)
@@ -1195,7 +1195,7 @@ class MDMFSlotReadProxy(object):
                  storage_server,
                  storage_index,
                  shnum,
-                 data="",
+                 data=b"",
                  data_is_everything=False):
         # Start the initialization process.
         self._storage_server = storage_server
@@ -1239,7 +1239,7 @@ class MDMFSlotReadProxy(object):
         # None if there isn't any cached data, but the way we index the
         # cached data requires a string, so convert None to "".
         if self._data == None:
-            self._data = ""
+            self._data = b""
 
 
     def _maybe_fetch_offsets_and_header(self, force_remote=False):
@@ -1417,7 +1417,7 @@ class MDMFSlotReadProxy(object):
                 # when we fetched the header
                 data = results[self.shnum]
                 if not data:
-                    data = ""
+                    data = b""
                 else:
                     if len(data) != 1:
                         raise BadShareError("got %d vectors, not 1" % len(data))
@@ -1426,7 +1426,7 @@ class MDMFSlotReadProxy(object):
             else:
                 data = results[self.shnum]
                 if not data:
-                    salt = data = ""
+                    salt = data = b""
                 else:
                     salt_and_data = results[self.shnum][0]
                     salt = salt_and_data[:SALT_SIZE]
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 6208af0fa..df625eda3 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -2409,7 +2409,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # since it will encounter them on the grid. Callers use the
         # is_sdmf method to test this.
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = mr.is_sdmf()
         d.addCallback(lambda issdmf:
             self.failUnless(issdmf))
@@ -2420,7 +2420,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # The slot read proxy should, naturally, know how to tell us
         # about data in the SDMF format
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.is_sdmf())
@@ -2491,7 +2491,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # read more segments than that. The reader should know this and
         # complain if we try to do that.
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.is_sdmf())
@@ -2513,7 +2513,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         mdmf_data = self.build_test_mdmf_share()
         self.write_test_share_to_server(b"si1")
         def _make_mr(ignored, length):
-            mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, mdmf_data[:length])
+            mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0, mdmf_data[:length])
             return mr
 
         d = defer.succeed(None)
@@ -2574,7 +2574,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         sdmf_data = self.build_test_sdmf_share()
         self.write_sdmf_share_to_server(b"si1")
         def _make_mr(ignored, length):
-            mr = MDMFSlotReadProxy(self.storage_server, "si1", 0, sdmf_data[:length])
+            mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0, sdmf_data[:length])
             return mr
 
         d = defer.succeed(None)
@@ -2640,7 +2640,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # unrelated to the actual handling of the content of the file.
         # The reader should behave intelligently in these cases.
         self.write_test_share_to_server(b"si1", empty=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct.
         d = defer.succeed(None)
@@ -2666,7 +2666,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_read_with_empty_sdmf_file(self):
         self.write_sdmf_share_to_server(b"si1", empty=True)
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the encoding parameters, and they
         # should be correct
         d = defer.succeed(None)
@@ -2692,7 +2692,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_verinfo_with_sdmf_file(self):
         self.write_sdmf_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         # We should be able to get the version information.
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
@@ -2733,7 +2733,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
 
     def test_verinfo_with_mdmf_file(self):
         self.write_test_share_to_server(b"si1")
-        mr = MDMFSlotReadProxy(self.storage_server, "si1", 0)
+        mr = MDMFSlotReadProxy(self.storage_server, b"si1", 0)
         d = defer.succeed(None)
         d.addCallback(lambda ignored:
             mr.get_verinfo())
@@ -2780,7 +2780,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         data = self.build_test_sdmf_share()
         sdmfr = SDMFSlotWriteProxy(0,
                                    self.storage_server,
-                                   "si1",
+                                   b"si1",
                                    self.secrets,
                                    0, 3, 10, 36, 36)
         # Put the block and salt.
@@ -2823,7 +2823,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         # don't, no write should occur.
         sdmfw = SDMFSlotWriteProxy(0,
                                    self.storage_server,
-                                   "si1",
+                                   b"si1",
                                    self.secrets,
                                    1, 3, 10, 36, 36)
         sdmfw.put_block(self.blockdata, 0, self.salt)
@@ -2845,7 +2845,7 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
         sdmfw.put_verification_key(self.verification_key)
 
         # We shouldn't have a checkstring yet
-        self.failUnlessEqual(sdmfw.get_checkstring(), "")
+        self.failUnlessEqual(sdmfw.get_checkstring(), b"")
 
         d = sdmfw.finish_publishing()
         def _then(results):

From 6c52a03030dcf67d22babd6304225aaee818fd35 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 11:59:34 -0400
Subject: [PATCH 07/13] Fix indent.

---
 src/allmydata/test/test_storage.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index df625eda3..3fc4fd11f 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1662,13 +1662,13 @@ class MDMFProxies(unittest.TestCase, ShouldFailMixin):
                               encprivkey_offset,
                               eof_offset)
         final_share = b"".join([prefix,
-                           offsets,
-                           self.verification_key,
-                           self.signature,
-                           self.share_hash_chain_s,
-                           self.block_hash_tree_s,
-                           block,
-                           self.encprivkey])
+                                offsets,
+                                self.verification_key,
+                                self.signature,
+                                self.share_hash_chain_s,
+                                self.block_hash_tree_s,
+                                block,
+                                self.encprivkey])
         self.offsets = {}
         self.offsets['signature'] = signature_offset
         self.offsets['share_hash_chain'] = sharehashes_offset

From 2b37da9ca027ade1d22e1a582d894600dbb6d884 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 11:59:39 -0400
Subject: [PATCH 08/13] filter() is lazy in Python 3.

---
 src/allmydata/mutable/layout.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py
index ee086ec5e..bf9a0483b 100644
--- a/src/allmydata/mutable/layout.py
+++ b/src/allmydata/mutable/layout.py
@@ -1744,7 +1744,7 @@ class MDMFSlotReadProxy(object):
 
 
     def _read(self, readvs, force_remote=False):
-        unsatisfiable = filter(lambda x: x[0] + x[1] > len(self._data), readvs)
+        unsatisfiable = list(filter(lambda x: x[0] + x[1] > len(self._data), readvs))
         # TODO: It's entirely possible to tweak this so that it just
         # fulfills the requests that it can, and not demand that all
         # requests are satisfiable before running it.

From 1cfe58a52dfae9682cb01aee8e8f077e919796be Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 13:17:52 -0400
Subject: [PATCH 09/13] All of test_storage passes on Python 3.

---
 src/allmydata/storage/immutable.py |  8 ++++++--
 src/allmydata/storage/server.py    | 11 ++++++++---
 src/allmydata/test/test_storage.py |  8 ++++----
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py
index 32ba1a739..8d83ec1b3 100644
--- a/src/allmydata/storage/immutable.py
+++ b/src/allmydata/storage/immutable.py
@@ -1,3 +1,5 @@
+from future.utils import bytes_to_native_str
+
 import os, stat, struct, time
 
 from foolscap.api import Referenceable
@@ -298,7 +300,9 @@ class BucketReader(Referenceable):
 
     def __repr__(self):
         return "<%s %s %s>" % (self.__class__.__name__,
-                               base32.b2a(self.storage_index[:8])[:12],
+                               bytes_to_native_str(
+                                   base32.b2a(self.storage_index[:8])[:12]
+                               ),
                                self.shnum)
 
     def remote_read(self, offset, length):
@@ -309,7 +313,7 @@ class BucketReader(Referenceable):
         return data
 
     def remote_advise_corrupt_share(self, reason):
-        return self.ss.remote_advise_corrupt_share("immutable",
+        return self.ss.remote_advise_corrupt_share(b"immutable",
                                                    self.storage_index,
                                                    self.shnum,
                                                    reason)
diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 387225b01..ad8001a60 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -1,3 +1,4 @@
+from future.utils import bytes_to_native_str
 import os, re, struct, time
 import weakref
 import six
@@ -676,6 +677,10 @@ class StorageServer(service.MultiService, Referenceable):
 
     def remote_advise_corrupt_share(self, share_type, storage_index, shnum,
                                     reason):
+        # This is a remote API, I believe, so this has to be bytes for legacy
+        # protocol backwards compatibility reasons.
+        assert isinstance(share_type, bytes)
+        assert isinstance(reason, bytes)
         fileutil.make_dirs(self.corruption_advisory_dir)
         now = time_format.iso_utc(sep="T")
         si_s = si_b2a(storage_index)
@@ -684,11 +689,11 @@ class StorageServer(service.MultiService, Referenceable):
                           "%s--%s-%d" % (now, si_s, shnum)).replace(":","")
         with open(fn, "w") as f:
             f.write("report: Share Corruption\n")
-            f.write("type: %s\n" % share_type)
-            f.write("storage_index: %s\n" % si_s)
+            f.write("type: %s\n" % bytes_to_native_str(share_type))
+            f.write("storage_index: %s\n" % bytes_to_native_str(si_s))
             f.write("share_number: %d\n" % shnum)
             f.write("\n")
-            f.write(reason)
+            f.write(bytes_to_native_str(reason))
             f.write("\n")
         log.msg(format=("client claims corruption in (%(share_type)s) " +
                         "%(si)s-%(shnum)d: %(reason)s"),
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 3fc4fd11f..5c8faabc3 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,4 @@
-from future.utils import native_str, PY3
+from future.utils import native_str, PY3, bytes_to_native_str
 
 import time
 import os.path
@@ -731,7 +731,7 @@ class Server(unittest.TestCase):
 
         si0_s = base32.b2a(b"si0")
         ss.remote_advise_corrupt_share(b"immutable", b"si0", 0,
-                                       "This share smells funny.\n")
+                                       b"This share smells funny.\n")
         reportdir = os.path.join(workdir, "corruption-advisories")
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 1)
@@ -755,11 +755,11 @@ class Server(unittest.TestCase):
 
         b = ss.remote_get_buckets(b"si1")
         self.failUnlessEqual(set(b.keys()), set([1]))
-        b[1].remote_advise_corrupt_share("This share tastes like dust.\n")
+        b[1].remote_advise_corrupt_share(b"This share tastes like dust.\n")
 
         reports = os.listdir(reportdir)
         self.failUnlessEqual(len(reports), 2)
-        report_si1 = [r for r in reports if si1_s in r][0]
+        report_si1 = [r for r in reports if bytes_to_native_str(si1_s) in r][0]
         f = open(os.path.join(reportdir, report_si1), "rb")
         report = f.read()
         f.close()

From 3fa919834ad0883e560cabfeb27df579186a0d83 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 31 Aug 2020 13:20:57 -0400
Subject: [PATCH 10/13] Finish port to Python 3.

---
 newsfragments/3397.minor           |  0
 src/allmydata/test/test_storage.py | 16 ++++++++++++----
 src/allmydata/util/_python3.py     |  1 +
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 newsfragments/3397.minor

diff --git a/newsfragments/3397.minor b/newsfragments/3397.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 5c8faabc3..072cc9127 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,4 +1,12 @@
-from future.utils import native_str, PY3, bytes_to_native_str
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import native_str, PY2, PY3, bytes_to_native_str
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 import time
 import os.path
@@ -392,7 +400,7 @@ class Server(unittest.TestCase):
 
     def test_remove_incoming(self):
         ss = self.create("test_remove_incoming")
-        already, writers = self.allocate(ss, b"vid", range(3), 10)
+        already, writers = self.allocate(ss, b"vid", list(range(3)), 10)
         for i,wb in writers.items():
             wb.remote_write(0, b"%10d" % i)
             wb.remote_close()
@@ -563,7 +571,7 @@ class Server(unittest.TestCase):
 
         # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and
         # 5000-1085=3915 free, therefore we can fit 39 100byte shares
-        already3, writers3 = self.allocate(ss, b"vid3", range(100), 100, canary)
+        already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary)
         self.failUnlessEqual(len(writers3), 39)
         self.failUnlessEqual(len(ss._active_writers), 39)
 
@@ -598,7 +606,7 @@ class Server(unittest.TestCase):
     def test_leases(self):
         ss = self.create("test_leases")
         canary = FakeCanary()
-        sharenums = range(5)
+        sharenums = list(range(5))
         size = 100
 
         rs0,cs0 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)),
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 683834673..afdbea1f0 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -92,6 +92,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_python3",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
+    "allmydata.test.test_storage",
     "allmydata.test.test_storage_web",
     "allmydata.test.test_time_format",
     "allmydata.test.test_uri",

From 6da338a86a2212b13396f074332133699a57846d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:09:35 -0400
Subject: [PATCH 11/13] Note it's been ported.

---
 src/allmydata/test/test_storage.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 072cc9127..797d02794 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -1,3 +1,8 @@
+"""
+Tests for allmydata.storage.
+
+Ported to Python 3.
+"""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

From 69575401569d6fb57d2f8df57a2ae389d0ed50ab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:10:13 -0400
Subject: [PATCH 12/13] Assert nodeid is bytes, to ease porting.

---
 src/allmydata/storage/server.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index ad8001a60..c044a9fb0 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -52,6 +52,7 @@ class StorageServer(service.MultiService, Referenceable):
         service.MultiService.__init__(self)
         assert isinstance(nodeid, bytes)
         assert len(nodeid) == 20
+        assert isinstance(nodeid, bytes)
         self.my_nodeid = nodeid
         self.storedir = storedir
         sharedir = os.path.join(storedir, "shares")

From fad93f4144bb2da4d629f6c36d5668628e989db9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Sep 2020 14:13:22 -0400
Subject: [PATCH 13/13] Use existing utility.

---
 src/allmydata/test/test_storage.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py
index 797d02794..4bbbcda30 100644
--- a/src/allmydata/test/test_storage.py
+++ b/src/allmydata/test/test_storage.py
@@ -8,7 +8,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from future.utils import native_str, PY2, PY3, bytes_to_native_str
+from future.utils import native_str, PY2, bytes_to_native_str
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
@@ -1373,8 +1373,7 @@ class MutableServer(unittest.TestCase):
         si = base32.b2a(b"si1")
         # note: this is a detail of the storage server implementation, and
         # may change in the future
-        if PY3:
-            si = si.decode("utf-8")
+        si = bytes_to_native_str(si)  # filesystem paths are native strings
         prefix = si[:2]
         prefixdir = os.path.join(self.workdir("test_remove"), "shares", prefix)
         bucketdir = os.path.join(prefixdir, si)