From 996a564c05b5360673c8d14d7697f7e153452443 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Mar 2021 10:50:44 -0500
Subject: [PATCH 01/13] Private key should be bytes.

---
 src/allmydata/test/matchers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py
index eaf7d13a5..2d5e74ffd 100644
--- a/src/allmydata/test/matchers.py
+++ b/src/allmydata/test/matchers.py
@@ -51,7 +51,7 @@ class MatchesNodePublicKey(object):
         :return Mismatch: If the keys don't match.
         """
         config = read_config(self.basedir, u"tub.port")
-        privkey_bytes = config.get_private_config("node.privkey")
+        privkey_bytes = config.get_private_config("node.privkey").encode("utf-8")
         private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0]
         signature = ed25519.sign_data(private_key, b"")
         other_public_key = ed25519.verifying_key_from_signing_key(other)

From 5ec52f9885f946e749496b2bea6158d7c9ef191f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Mar 2021 14:14:55 -0500
Subject: [PATCH 02/13] More passing tests on Python 3.

---
 src/allmydata/test/test_client.py | 41 +++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 63a5ceaaa..07dac634b 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -1,3 +1,5 @@
+from past.builtins import unicode
+
 import os, sys
 from functools import (
     partial,
@@ -186,7 +188,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
             basedir,
             "client.port",
         )
-        abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding())
+        abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir))
         self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
@@ -234,7 +236,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"),
                        BASECONFIG)
         c = yield client.create_client(basedir)
-        self.failUnless(c.get_long_nodeid().startswith("v0-"))
+        self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
 
     @defer.inlineCallbacks
     def test_nodekey_no_storage(self):
@@ -246,7 +248,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"),
                        BASECONFIG + "[storage]\n" + "enabled = false\n")
         c = yield client.create_client(basedir)
-        self.failUnless(c.get_long_nodeid().startswith("v0-"))
+        self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
 
     def test_storage_anonymous_enabled_by_default(self):
         """
@@ -431,6 +433,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         generic helper for following storage_dir tests
         """
+        assert isinstance(basedir, unicode)
+        assert isinstance(storage_path, (unicode, type(None)))
+        assert isinstance(expected_path, unicode)
         os.mkdir(basedir)
         cfg_path = os.path.join(basedir, "tahoe.cfg")
         fileutil.write(
@@ -504,7 +509,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         expected_path = abspath_expanduser_unicode(
             u"client.Basic.test_absolute_storage_dir_myowndir/" + base
         )
-        config_path = expected_path.encode("utf-8")
+        config_path = expected_path
         return self._storage_dir_test(
             basedir,
             config_path,
@@ -516,32 +521,42 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
 
     def test_permute(self):
         sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
-        for k in ["%d" % i for i in range(5)]:
+        ks = [b"%d" % i for i in range(5)]
+        for k in ks:
             ann = {"anonymous-storage-FURL": SOME_FURL,
                    "permutation-seed-base32": base32.b2a(k) }
             sb.test_add_rref(k, "rref", ann)
 
-        self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2'])
-        self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3'])
+        one = self._permute(sb, b"one")
+        two = self._permute(sb, b"two")
+        self.assertEqual(sorted(one), ks)
+        self.assertEqual(sorted(two), ks)
+        self.assertNotEqual(one, two)
         sb.servers.clear()
-        self.failUnlessReallyEqual(self._permute(sb, "one"), [])
+        self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
 
     def test_permute_with_preferred(self):
         sb = StorageFarmBroker(
             True,
             None,
             EMPTY_CLIENT_CONFIG,
-            StorageClientConfig(preferred_peers=['1','4']),
+            StorageClientConfig(preferred_peers=[b'1',b'4']),
         )
-        for k in ["%d" % i for i in range(5)]:
+        ks = [b"%d" % i for i in range(5)]
+        for k in [b"%d" % i for i in range(5)]:
             ann = {"anonymous-storage-FURL": SOME_FURL,
                    "permutation-seed-base32": base32.b2a(k) }
             sb.test_add_rref(k, "rref", ann)
 
-        self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2'])
-        self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3'])
+        one = self._permute(sb, b"one")
+        two = self._permute(sb, b"two")
+        self.assertEqual(sorted(one), ks)
+        self.assertEqual(sorted(one[:2]), [b"1", b"4"])
+        self.assertEqual(sorted(two), ks)
+        self.assertEqual(sorted(two[:2]), [b"1", b"4"])
+        self.assertNotEqual(one, two)
         sb.servers.clear()
-        self.failUnlessReallyEqual(self._permute(sb, "one"), [])
+        self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
 
     @defer.inlineCallbacks
     def test_versions(self):

From 2c3353f2fa2ed0863f5edbed495478dd088a1804 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 3 Mar 2021 14:42:55 -0500
Subject: [PATCH 03/13] More tests passing on Python 3.

---
 src/allmydata/client.py           |  6 +++---
 src/allmydata/test/test_client.py | 32 +++++++++++++++----------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index f5e603490..7f640a8fb 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -364,8 +364,8 @@ class _StoragePlugins(object):
         """
         return set(
             config.get_config(
-                "storage", "plugins", b""
-            ).decode("ascii").split(u",")
+                "storage", "plugins", ""
+            ).split(u",")
         ) - {u""}
 
     @classmethod
@@ -870,7 +870,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         """
         Register a storage server.
         """
-        config_key = b"storage-plugin.{}.furl".format(
+        config_key = "storage-plugin.{}.furl".format(
             # Oops, why don't I have a better handle on this value?
             announceable_storage_server.announcement[u"name"],
         )
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 07dac634b..249be7c38 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -482,7 +482,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         the node's basedir.
         """
         basedir = u"client.Basic.test_relative_storage_dir"
-        config_path = b"myowndir"
+        config_path = u"myowndir"
         expected_path = os.path.join(
             abspath_expanduser_unicode(basedir),
             u"myowndir",
@@ -572,8 +572,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         c = yield client.create_client(basedir)
         ss = c.getServiceNamed("storage")
         verdict = ss.remote_get_version()
-        self.failUnlessReallyEqual(verdict["application-version"],
-                                   str(allmydata.__full_version__))
+        self.failUnlessReallyEqual(verdict[b"application-version"],
+                                   allmydata.__full_version__.encode("ascii"))
         self.failIfEqual(str(allmydata.__version__), "unknown")
         self.failUnless("." in str(allmydata.__full_version__),
                         "non-numeric version in '%s'" % allmydata.__version__)
@@ -918,7 +918,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
         private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG.encode("ascii"))
         basedir.child(client._Client.EXIT_TRIGGER_FILE).touch()
         yield client.create_client(basedir.path)
 
@@ -929,7 +929,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
         private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii"))
         c1 = yield client.create_client(basedir.path)
         c1.setServiceParent(self.sparent)
 
@@ -1056,7 +1056,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
         c = yield client.create_client(basedir)
 
-        n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+        n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failUnless(IImmutableFileNode.providedBy(n))
@@ -1074,10 +1074,10 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         # current fix for this (hopefully to be superceded by a better fix
         # eventually) is to prevent re-use of filenodes, so the NodeMaker is
         # hereby required *not* to cache and re-use filenodes for CHKs.
-        other_n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+        other_n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
         self.failIf(n is other_n, (n, other_n))
 
-        n = c.create_node_from_uri("URI:LIT:n5xgk")
+        n = c.create_node_from_uri(b"URI:LIT:n5xgk")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failUnless(IImmutableFileNode.providedBy(n))
@@ -1086,7 +1086,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failIf(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1095,7 +1095,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failIf(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1104,7 +1104,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1113,7 +1113,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failIf(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1122,8 +1122,8 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        unknown_rw = "lafs://from_the_future"
-        unknown_ro = "lafs://readonly_from_the_future"
+        unknown_rw = b"lafs://from_the_future"
+        unknown_ro = b"lafs://readonly_from_the_future"
         n = c.create_node_from_uri(unknown_rw, unknown_ro)
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
@@ -1133,7 +1133,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_unknown())
         self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
         self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
-        self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
+        self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)
 
         # Note: it isn't that we *intend* to deploy non-ASCII caps in
         # the future, it is that we want to make sure older Tahoe-LAFS
@@ -1150,7 +1150,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_unknown())
         self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
         self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
-        self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
+        self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)
 
 
 

From cf418b753a5cf1762d431b317d32e73bb96174af Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 4 Mar 2021 10:51:24 -0500
Subject: [PATCH 04/13] All tests pass on Python 3.

---
 src/allmydata/storage_client.py   |  2 ++
 src/allmydata/test/test_client.py | 11 ++++++-----
 src/allmydata/util/eliotutil.py   | 10 +++++++++-
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 75c554562..b59a7f1bc 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -37,6 +37,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_text
 
 import re, time, hashlib
 
@@ -198,6 +199,7 @@ class StorageFarmBroker(service.MultiService):
         # doesn't really matter but it makes the logging behavior more
         # predictable and easier to test (and at least one test does depend on
         # this sorted order).
+        servers = {ensure_text(key): value for (key, value) in servers.items()}
         for (server_id, server) in sorted(servers.items()):
             try:
                 storage_server = self._make_storage_server(
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 249be7c38..fdebf4ac2 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -1,3 +1,4 @@
+from future.utils import PY2
 from past.builtins import unicode
 
 import os, sys
@@ -23,7 +24,6 @@ from hypothesis.strategies import (
 )
 
 from eliot.testing import (
-    capture_logging,
     assertHasAction,
 )
 from twisted.trial import unittest
@@ -64,6 +64,7 @@ from allmydata.util import (
     encodingutil,
     configutil,
 )
+from allmydata.util.eliotutil import capture_logging
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.interfaces import IFilesystemNode, IFileNode, \
      IImmutableFileNode, IMutableFileNode, IDirectoryNode
@@ -798,7 +799,7 @@ class StaticServers(Fixture):
                 for (serverid, announcement)
                 in self._server_details
             },
-        }))
+        }).encode("utf-8"))
 
 
 class StorageClients(SyncTestCase):
@@ -847,7 +848,7 @@ class StorageClients(SyncTestCase):
             succeeded(
                 AfterPreprocessing(
                     get_known_server_details,
-                    Equals([(serverid, announcement)]),
+                    Equals([(serverid.encode("utf-8"), announcement)]),
                 ),
             ),
         )
@@ -874,7 +875,7 @@ class StorageClients(SyncTestCase):
         self.useFixture(
             StaticServers(
                 self.basedir,
-                [(serverid, announcement),
+                [(serverid.encode("ascii"), announcement),
                  # Along with a "bad" server announcement.  Order in this list
                  # doesn't matter, yaml serializer and Python dicts are going
                  # to shuffle everything around kind of randomly.
@@ -891,7 +892,7 @@ class StorageClients(SyncTestCase):
                 AfterPreprocessing(
                     get_known_server_details,
                     # It should have the good server details.
-                    Equals([(serverid, announcement)]),
+                    Equals([(serverid.encode("utf-8"), announcement)]),
                 ),
             ),
         )
diff --git a/src/allmydata/util/eliotutil.py b/src/allmydata/util/eliotutil.py
index 9e3cdd3e1..5d144eb1d 100644
--- a/src/allmydata/util/eliotutil.py
+++ b/src/allmydata/util/eliotutil.py
@@ -32,7 +32,7 @@ from six import ensure_text
 from sys import (
     stdout,
 )
-from functools import wraps
+from functools import wraps, partial
 from logging import (
     INFO,
     Handler,
@@ -66,6 +66,7 @@ from eliot.twisted import (
     DeferredContext,
     inline_callbacks,
 )
+from eliot.testing import capture_logging as eliot_capture_logging
 
 from twisted.python.usage import (
     UsageError,
@@ -326,3 +327,10 @@ def log_call_deferred(action_type):
                 return DeferredContext(d).addActionFinish()
         return logged_f
     return decorate_log_call_deferred
+
+# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON
+# encoder we want to use when validating messages.
+if PY2:
+    capture_logging = eliot_capture_logging
+else:
+    capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder)

From a29c03d34b2c93920f16993abaac1b2b9a93f183 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 4 Mar 2021 11:02:36 -0500
Subject: [PATCH 05/13] Port to Python 3.

---
 src/allmydata/test/test_client.py | 19 ++++++++++++++-----
 src/allmydata/util/_python3.py    |  1 +
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index fdebf4ac2..3d48d7eef 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -1,5 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from future.utils import PY2
-from past.builtins import unicode
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, sys
 from functools import (
@@ -189,7 +198,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
             basedir,
             "client.port",
         )
-        abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir))
+        abs_basedir = fileutil.abspath_expanduser_unicode(str(basedir))
         self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
@@ -434,9 +443,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         generic helper for following storage_dir tests
         """
-        assert isinstance(basedir, unicode)
-        assert isinstance(storage_path, (unicode, type(None)))
-        assert isinstance(expected_path, unicode)
+        assert isinstance(basedir, str)
+        assert isinstance(storage_path, (str, type(None)))
+        assert isinstance(expected_path, str)
         os.mkdir(basedir)
         cfg_path = os.path.join(basedir, "tahoe.cfg")
         fileutil.write(
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 39bcd83eb..48df3d5d5 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -151,6 +151,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
     "allmydata.test.test_checker",
+    "allmydata.test.test_client",
     "allmydata.test.test_codec",
     "allmydata.test.test_common_util",
     "allmydata.test.test_configutil",

From f2970f1b161eb43ed20fe619bdfe755942f006b9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 5 Mar 2021 10:19:24 -0500
Subject: [PATCH 06/13] Port to Python 3.

---
 src/allmydata/client.py        | 18 ++++++++++++++----
 src/allmydata/util/_python3.py |  1 +
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 7f640a8fb..7e93bc0dd 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -1,4 +1,14 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode
@@ -460,7 +470,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
 
     introducers = config.get_introducer_configuration()
 
-    for petname, (furl, cache_path) in introducers.items():
+    for petname, (furl, cache_path) in list(introducers.items()):
         ic = _introducer_factory(
             main_tub,
             furl.encode("ascii"),
@@ -679,7 +689,7 @@ class _Client(node.Node, pollmixin.PollMixin):
     def init_secrets(self):
         # configs are always unicode
         def _unicode_make_secret():
-            return unicode(_make_secret(), "ascii")
+            return str(_make_secret(), "ascii")
         lease_s = self.config.get_or_create_private_config(
             "secret", _unicode_make_secret).encode("utf-8")
         lease_secret = base32.a2b(lease_s)
@@ -694,7 +704,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         def _make_key():
             private_key, _ = ed25519.create_signing_keypair()
             # Config values are always unicode:
-            return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
+            return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
 
         private_key_str = self.config.get_or_create_private_config(
             "node.privkey", _make_key).encode("utf-8")
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 48df3d5d5..82ae41e4e 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -29,6 +29,7 @@ PORTED_MODULES = [
     "allmydata._monkeypatch",
     "allmydata.blacklist",
     "allmydata.check_results",
+    "allmydata.client",
     "allmydata.codec",
     "allmydata.control",
     "allmydata.crypto",

From c71fa48f3cb852db6de1724b83172912c4e584b9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 5 Mar 2021 10:19:48 -0500
Subject: [PATCH 07/13] News file.

---
 newsfragments/3625.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3625.minor

diff --git a/newsfragments/3625.minor b/newsfragments/3625.minor
new file mode 100644
index 000000000..e69de29bb

From d6fc3c078e421207fef8719c295ec009c441958e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 5 Mar 2021 10:26:38 -0500
Subject: [PATCH 08/13] Port to Python 3.

---
 src/allmydata/test/matchers.py | 10 ++++++++++
 src/allmydata/util/_python3.py |  1 +
 2 files changed, 11 insertions(+)

diff --git a/src/allmydata/test/matchers.py b/src/allmydata/test/matchers.py
index 2d5e74ffd..3359a7ed5 100644
--- a/src/allmydata/test/matchers.py
+++ b/src/allmydata/test/matchers.py
@@ -1,6 +1,16 @@
 """
 Testtools-style matchers useful to the Tahoe-LAFS test suite.
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import attr
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 82ae41e4e..aa7323900 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -87,6 +87,7 @@ PORTED_MODULES = [
     "allmydata.storage.server",
     "allmydata.storage.shares",
     "allmydata.test.no_network",
+    "allmydata.test.matchers",
     "allmydata.test.mutable.util",
     "allmydata.unknown",
     "allmydata.uri",

From 2b9e1996ec2773ea873a2665110fe565fdb109e3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 5 Mar 2021 10:34:56 -0500
Subject: [PATCH 09/13] Fix Python 2 regression.

---
 src/allmydata/client.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 7e93bc0dd..3bf976fe5 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -8,7 +8,9 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+    # Don't use future str to prevent leaking future's newbytes into foolscap, which they break.
+    from past.builtins import unicode as str
 
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode

From 226b6195217a2417b1d167fa3493bb8720095629 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 9 Mar 2021 09:52:48 -0500
Subject: [PATCH 10/13] Don't delete test_client.py.

---
 nix/tahoe-lafs.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index f3ccf950d..c0262768c 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -28,7 +28,6 @@ python.pkgs.buildPythonPackage rec {
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
-    rm src/allmydata/test/test_client.py
   '';
 
 

From bc7a9a4b7e53063a92c466650ebce9f84a883df8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 9 Mar 2021 10:38:49 -0500
Subject: [PATCH 11/13] Try to fix Nix a different way.

---
 nix/tahoe-lafs.nix | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index c0262768c..c831b6b7b 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -24,10 +24,17 @@ python.pkgs.buildPythonPackage rec {
     # tests with in a module.
 
     # Many of these tests don't properly skip when i2p or tor dependencies are
-    # not supplied (and we are not supplying them).
+    # not supplied (and we are not supplying them). test_client.py fails because
+    # version is "unknown" on Nix.
+    # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter.
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
+    rm src/allmydata/test/test_client.py
+
+    # Since we're deleting files, this complains they're missing. For now Nix
+    # is Python 2-only, anyway, so these tests don't add anything yet.
+    rm src/allmydata/test/test_python3.py
   '';
 
 

From 6e9a3fa3e9f2cf81e3a28aa895fea784d5074922 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 10:08:18 -0400
Subject: [PATCH 12/13] Add back hard-coded expected results.

---
 src/allmydata/test/test_client.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 3d48d7eef..957be5197 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -530,6 +530,16 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         return [ s.get_longname() for s in sb.get_servers_for_psi(key) ]
 
     def test_permute(self):
+        """
+        Permutations need to be stable across Tahoe releases, which is why we
+        hardcode a specific expected order.
+
+        This is because the order of these results determines which servers a
+        client will choose to place shares on and which servers it will consult
+        (and in what order) when trying to retrieve those shares.  If the order
+        ever changes, all already-placed shares become (at best) harder to find
+        or (at worst) impossible to find.
+        """
         sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
         ks = [b"%d" % i for i in range(5)]
         for k in ks:
@@ -539,6 +549,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
 
         one = self._permute(sb, b"one")
         two = self._permute(sb, b"two")
+        self.failUnlessReallyEqual(one, [b'3',b'1',b'0',b'4',b'2'])
+        self.failUnlessReallyEqual(two, [b'0',b'4',b'2',b'1',b'3'])
         self.assertEqual(sorted(one), ks)
         self.assertEqual(sorted(two), ks)
         self.assertNotEqual(one, two)
@@ -546,6 +558,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
 
     def test_permute_with_preferred(self):
+        """
+        Permutations need to be stable across Tahoe releases, which is why we
+        hardcode a specific expected order.  In this case, two values are
+        preferred and should come first.
+        """
         sb = StorageFarmBroker(
             True,
             None,
@@ -560,6 +577,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
 
         one = self._permute(sb, b"one")
         two = self._permute(sb, b"two")
+        self.failUnlessReallyEqual(b"".join(one), b'14302')
+        self.failUnlessReallyEqual(b"".join(two), b'41023')
         self.assertEqual(sorted(one), ks)
         self.assertEqual(sorted(one[:2]), [b"1", b"4"])
         self.assertEqual(sorted(two), ks)

From 2eba96de61e89f3b836a38ff288790893ba21e30 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 17 Mar 2021 15:27:45 -0400
Subject: [PATCH 13/13] Address review comments.

---
 src/allmydata/scripts/common.py   | 5 ++---
 src/allmydata/scripts/runner.py   | 5 +++--
 src/allmydata/test/test_runner.py | 9 ++++-----
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index bcaa84649..36330f535 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 
 from __future__ import print_function
+from six import ensure_str
 
 import os, sys, textwrap
 import codecs
@@ -286,7 +287,5 @@ def escape_path(path):
     """
     segments = path.split("/")
     result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])
-    # fixme: test.cli.test_create_alias fails if it gets Unicode on Python 2
-    if PY2 and isinstance(result, type(u'')):
-        result = result.encode('ascii')
+    result = ensure_str(result, "ascii")
     return result
diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index 705fa37bf..ada3c2dfc 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -174,8 +174,9 @@ def _maybe_enable_eliot_logging(options, reactor):
     return options
 
 def run():
-    if not six.PY2:
-        warnings.warn("Support for Python 3 is experimental. Use at your own risk.")
+    if six.PY3:
+        warnings.warn("Support for Python 3 is an incomplete work-in-progress."
+                      " Use at your own risk.")
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize
diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index e7e9892fb..f6a7c2ee1 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -12,6 +12,9 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from six import ensure_text
+
 import os.path, re, sys
 from os import linesep
 
@@ -19,8 +22,6 @@ from eliot import (
     log_call,
 )
 
-import six
-
 from twisted.trial import unittest
 
 from twisted.internet import reactor
@@ -83,9 +84,7 @@ def run_bintahoe(extra_argv, python_options=None):
     :return: A three-tuple of stdout (unicode), stderr (unicode), and the
         child process "returncode" (int).
     """
-    # fixme: below, 'unicode_to_argv' is called so ensure that
-    # executable is unicode to support that expectation.
-    executable = sys.executable.decode('utf-8') if six.PY2 else sys.executable
+    executable = ensure_text(sys.executable)
     argv = [executable]
     if python_options is not None:
         argv.extend(python_options)