Merge remote-tracking branch 'origin/master' into 3603.scripts

newsfragments/3633.installation

@@ -0,0 +1 @@
+Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support.

nix/tahoe-lafs.nix

@@ -24,11 +24,17 @@ python.pkgs.buildPythonPackage rec {
     # tests with in a module.
 
     # Many of these tests don't properly skip when i2p or tor dependencies are
-    # not supplied (and we are not supplying them).
+    # not supplied (and we are not supplying them). test_client.py fails because
+    # version is "unknown" on Nix.
+    # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter.
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
     rm src/allmydata/test/test_client.py
+
+    # Since we're deleting files, this complains they're missing. For now Nix
+    # is Python 2-only, anyway, so these tests don't add anything yet.
+    rm src/allmydata/test/test_python3.py
   '';
 
 

setup.py

@@ -151,8 +151,13 @@ tor_requires = [
 ]
 
 i2p_requires = [
-    # txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10.
+    # txi2p has Python 3 support in master branch, but it has not been
+    # released -- see https://github.com/str4d/txi2p/issues/10.  We
+    # could use a fork for Python 3 until txi2p's maintainers are back
+    # in action.  For Python 2, we could continue using the txi2p
+    # version about which no one has complained to us so far.
     "txi2p; python_version < '3.0'",
+    "txi2p-tahoe >= 0.3.5; python_version > '3.0'",
 ]
 
 if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency':

src/allmydata/client.py

@@ -1,4 +1,16 @@
-from past.builtins import unicode
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+    # Don't use future str to prevent leaking future's newbytes into foolscap, which they break.
+    from past.builtins import unicode as str
 
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode
@@ -364,8 +376,8 @@ class _StoragePlugins(object):
         """
         return set(
             config.get_config(
-                "storage", "plugins", b""
+                "storage", "plugins", ""
-            ).decode("ascii").split(u",")
+            ).split(u",")
         ) - {u""}
 
     @classmethod
@@ -460,7 +472,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
 
     introducers = config.get_introducer_configuration()
 
-    for petname, (furl, cache_path) in introducers.items():
+    for petname, (furl, cache_path) in list(introducers.items()):
         ic = _introducer_factory(
             main_tub,
             furl.encode("ascii"),
@@ -679,7 +691,7 @@ class _Client(node.Node, pollmixin.PollMixin):
     def init_secrets(self):
         # configs are always unicode
         def _unicode_make_secret():
-            return unicode(_make_secret(), "ascii")
+            return str(_make_secret(), "ascii")
         lease_s = self.config.get_or_create_private_config(
             "secret", _unicode_make_secret).encode("utf-8")
         lease_secret = base32.a2b(lease_s)
@@ -694,7 +706,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         def _make_key():
             private_key, _ = ed25519.create_signing_keypair()
             # Config values are always unicode:
-            return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
+            return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
 
         private_key_str = self.config.get_or_create_private_config(
             "node.privkey", _make_key).encode("utf-8")
@@ -870,7 +882,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         """
         Register a storage server.
         """
-        config_key = b"storage-plugin.{}.furl".format(
+        config_key = "storage-plugin.{}.furl".format(
             # Oops, why don't I have a better handle on this value?
             announceable_storage_server.announcement[u"name"],
         )

src/allmydata/storage_client.py

@@ -37,6 +37,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_text
 
 import re, time, hashlib
 
@@ -198,6 +199,7 @@ class StorageFarmBroker(service.MultiService):
         # doesn't really matter but it makes the logging behavior more
         # predictable and easier to test (and at least one test does depend on
         # this sorted order).
+        servers = {ensure_text(key): value for (key, value) in servers.items()}
         for (server_id, server) in sorted(servers.items()):
             try:
                 storage_server = self._make_storage_server(

src/allmydata/test/cli/common.py

@@ -52,6 +52,8 @@ class CLITestMixin(ReallyEqualMixin):
         # Python functions want native strings.  So ignore the requirements
         # for passing arguments to another process and make sure this argument
         # is a native string.
+        verb = ensure_str(verb)
+        args = [ensure_str(arg) for arg in args]
         client_dir = ensure_str(self.get_clientdir(i=client_num))
         nodeargs = [ b"--node-directory", client_dir ]
         return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)

src/allmydata/test/common.py

@@ -1059,7 +1059,7 @@ def _corrupt_mutable_share_data(data, debug=False):
     assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
     data_offset = MutableShareFile.DATA_OFFSET
     sharetype = data[data_offset:data_offset+1]
-    assert sharetype == "\x00", "non-SDMF mutable shares not supported"
+    assert sharetype == b"\x00", "non-SDMF mutable shares not supported"
     (version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
      ig_datalen, offsets) = unpack_header(data[data_offset:])
     assert version == 0, "this function only handles v0 SDMF files"

src/allmydata/test/matchers.py

@@ -1,6 +1,16 @@
 """
 Testtools-style matchers useful to the Tahoe-LAFS test suite.
+
+Ported to Python 3.
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import attr
 
@@ -51,7 +61,7 @@ class MatchesNodePublicKey(object):
         :return Mismatch: If the keys don't match.
         """
         config = read_config(self.basedir, u"tub.port")
-        privkey_bytes = config.get_private_config("node.privkey")
+        privkey_bytes = config.get_private_config("node.privkey").encode("utf-8")
         private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0]
         signature = ed25519.sign_data(private_key, b"")
         other_public_key = ed25519.verifying_key_from_signing_key(other)

src/allmydata/test/test_client.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os, sys
 from functools import (
     partial,
@@ -21,7 +33,6 @@ from hypothesis.strategies import (
 )
 
 from eliot.testing import (
-    capture_logging,
     assertHasAction,
 )
 from twisted.trial import unittest
@@ -62,6 +73,7 @@ from allmydata.util import (
     encodingutil,
     configutil,
 )
+from allmydata.util.eliotutil import capture_logging
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.interfaces import IFilesystemNode, IFileNode, \
      IImmutableFileNode, IMutableFileNode, IDirectoryNode
@@ -186,7 +198,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
             basedir,
             "client.port",
         )
-        abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding())
+        abs_basedir = fileutil.abspath_expanduser_unicode(str(basedir))
         self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
         self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
@@ -234,7 +246,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"),
                        BASECONFIG)
         c = yield client.create_client(basedir)
-        self.failUnless(c.get_long_nodeid().startswith("v0-"))
+        self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
 
     @defer.inlineCallbacks
     def test_nodekey_no_storage(self):
@@ -246,7 +258,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"),
                        BASECONFIG + "[storage]\n" + "enabled = false\n")
         c = yield client.create_client(basedir)
-        self.failUnless(c.get_long_nodeid().startswith("v0-"))
+        self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
 
     def test_storage_anonymous_enabled_by_default(self):
         """
@@ -431,6 +443,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         generic helper for following storage_dir tests
         """
+        assert isinstance(basedir, str)
+        assert isinstance(storage_path, (str, type(None)))
+        assert isinstance(expected_path, str)
         os.mkdir(basedir)
         cfg_path = os.path.join(basedir, "tahoe.cfg")
         fileutil.write(
@@ -477,7 +492,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         the node's basedir.
         """
         basedir = u"client.Basic.test_relative_storage_dir"
-        config_path = b"myowndir"
+        config_path = u"myowndir"
         expected_path = os.path.join(
             abspath_expanduser_unicode(basedir),
             u"myowndir",
@@ -504,7 +519,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         expected_path = abspath_expanduser_unicode(
             u"client.Basic.test_absolute_storage_dir_myowndir/" + base
         )
-        config_path = expected_path.encode("utf-8")
+        config_path = expected_path
         return self._storage_dir_test(
             basedir,
             config_path,
@@ -515,33 +530,62 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         return [ s.get_longname() for s in sb.get_servers_for_psi(key) ]
 
     def test_permute(self):
+        """
+        Permutations need to be stable across Tahoe releases, which is why we
+        hardcode a specific expected order.
+
+        This is because the order of these results determines which servers a
+        client will choose to place shares on and which servers it will consult
+        (and in what order) when trying to retrieve those shares.  If the order
+        ever changes, all already-placed shares become (at best) harder to find
+        or (at worst) impossible to find.
+        """
         sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
-        for k in ["%d" % i for i in range(5)]:
+        ks = [b"%d" % i for i in range(5)]
+        for k in ks:
             ann = {"anonymous-storage-FURL": SOME_FURL,
                    "permutation-seed-base32": base32.b2a(k) }
             sb.test_add_rref(k, "rref", ann)
 
-        self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2'])
+        one = self._permute(sb, b"one")
-        self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3'])
+        two = self._permute(sb, b"two")
+        self.failUnlessReallyEqual(one, [b'3',b'1',b'0',b'4',b'2'])
+        self.failUnlessReallyEqual(two, [b'0',b'4',b'2',b'1',b'3'])
+        self.assertEqual(sorted(one), ks)
+        self.assertEqual(sorted(two), ks)
+        self.assertNotEqual(one, two)
         sb.servers.clear()
-        self.failUnlessReallyEqual(self._permute(sb, "one"), [])
+        self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
 
     def test_permute_with_preferred(self):
+        """
+        Permutations need to be stable across Tahoe releases, which is why we
+        hardcode a specific expected order.  In this case, two values are
+        preferred and should come first.
+        """
         sb = StorageFarmBroker(
             True,
             None,
             EMPTY_CLIENT_CONFIG,
-            StorageClientConfig(preferred_peers=['1','4']),
+            StorageClientConfig(preferred_peers=[b'1',b'4']),
         )
-        for k in ["%d" % i for i in range(5)]:
+        ks = [b"%d" % i for i in range(5)]
+        for k in [b"%d" % i for i in range(5)]:
             ann = {"anonymous-storage-FURL": SOME_FURL,
                    "permutation-seed-base32": base32.b2a(k) }
             sb.test_add_rref(k, "rref", ann)
 
-        self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2'])
+        one = self._permute(sb, b"one")
-        self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3'])
+        two = self._permute(sb, b"two")
+        self.failUnlessReallyEqual(b"".join(one), b'14302')
+        self.failUnlessReallyEqual(b"".join(two), b'41023')
+        self.assertEqual(sorted(one), ks)
+        self.assertEqual(sorted(one[:2]), [b"1", b"4"])
+        self.assertEqual(sorted(two), ks)
+        self.assertEqual(sorted(two[:2]), [b"1", b"4"])
+        self.assertNotEqual(one, two)
         sb.servers.clear()
-        self.failUnlessReallyEqual(self._permute(sb, "one"), [])
+        self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
 
     @defer.inlineCallbacks
     def test_versions(self):
@@ -557,8 +601,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         c = yield client.create_client(basedir)
         ss = c.getServiceNamed("storage")
         verdict = ss.remote_get_version()
-        self.failUnlessReallyEqual(verdict["application-version"],
+        self.failUnlessReallyEqual(verdict[b"application-version"],
-                                   str(allmydata.__full_version__))
+                                   allmydata.__full_version__.encode("ascii"))
         self.failIfEqual(str(allmydata.__version__), "unknown")
         self.failUnless("." in str(allmydata.__full_version__),
                         "non-numeric version in '%s'" % allmydata.__version__)
@@ -783,7 +827,7 @@ class StaticServers(Fixture):
                 for (serverid, announcement)
                 in self._server_details
             },
-        }))
+        }).encode("utf-8"))
 
 
 class StorageClients(SyncTestCase):
@@ -832,7 +876,7 @@ class StorageClients(SyncTestCase):
             succeeded(
                 AfterPreprocessing(
                     get_known_server_details,
-                    Equals([(serverid, announcement)]),
+                    Equals([(serverid.encode("utf-8"), announcement)]),
                 ),
             ),
         )
@@ -859,7 +903,7 @@ class StorageClients(SyncTestCase):
         self.useFixture(
             StaticServers(
                 self.basedir,
-                [(serverid, announcement),
+                [(serverid.encode("ascii"), announcement),
                  # Along with a "bad" server announcement.  Order in this list
                  # doesn't matter, yaml serializer and Python dicts are going
                  # to shuffle everything around kind of randomly.
@@ -876,7 +920,7 @@ class StorageClients(SyncTestCase):
                 AfterPreprocessing(
                     get_known_server_details,
                     # It should have the good server details.
-                    Equals([(serverid, announcement)]),
+                    Equals([(serverid.encode("utf-8"), announcement)]),
                 ),
             ),
         )
@@ -903,7 +947,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
         private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG.encode("ascii"))
         basedir.child(client._Client.EXIT_TRIGGER_FILE).touch()
         yield client.create_client(basedir.path)
 
@@ -914,7 +958,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
         private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii"))
         c1 = yield client.create_client(basedir.path)
         c1.setServiceParent(self.sparent)
 
@@ -1041,7 +1085,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
         c = yield client.create_client(basedir)
 
-        n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+        n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failUnless(IImmutableFileNode.providedBy(n))
@@ -1059,10 +1103,10 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         # current fix for this (hopefully to be superceded by a better fix
         # eventually) is to prevent re-use of filenodes, so the NodeMaker is
         # hereby required *not* to cache and re-use filenodes for CHKs.
-        other_n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
+        other_n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
         self.failIf(n is other_n, (n, other_n))
 
-        n = c.create_node_from_uri("URI:LIT:n5xgk")
+        n = c.create_node_from_uri(b"URI:LIT:n5xgk")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failUnless(IImmutableFileNode.providedBy(n))
@@ -1071,7 +1115,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failIf(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1080,7 +1124,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failIf(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failUnless(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1089,7 +1133,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1098,7 +1142,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failIf(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
+        n = c.create_node_from_uri(b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
         self.failIf(IImmutableFileNode.providedBy(n))
@@ -1107,8 +1151,8 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_readonly())
         self.failUnless(n.is_mutable())
 
-        unknown_rw = "lafs://from_the_future"
+        unknown_rw = b"lafs://from_the_future"
-        unknown_ro = "lafs://readonly_from_the_future"
+        unknown_ro = b"lafs://readonly_from_the_future"
         n = c.create_node_from_uri(unknown_rw, unknown_ro)
         self.failUnless(IFilesystemNode.providedBy(n))
         self.failIf(IFileNode.providedBy(n))
@@ -1118,7 +1162,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_unknown())
         self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
         self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
-        self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
+        self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)
 
         # Note: it isn't that we *intend* to deploy non-ASCII caps in
         # the future, it is that we want to make sure older Tahoe-LAFS
@@ -1135,7 +1179,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
         self.failUnless(n.is_unknown())
         self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
         self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
-        self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
+        self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)
 
 
 

src/allmydata/test/test_deepcheck.py

@@ -1,4 +1,10 @@
-import os, json, urllib
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 # Python 2 compatibility
 # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
@@ -11,7 +17,13 @@ import os, json, urllib
 # (Pdb) pp data
 # '334:12:b\'mutable-good\',90:URI:SSK-RO:...
 from past.builtins import unicode as str
-from future.utils import native_str
+from future.utils import PY3, PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
+
+
+import os, json
+from urllib.parse import quote as url_quote
 
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -29,31 +41,37 @@ from allmydata.uri import LiteralFileURI
 
 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
      ShouldFailMixin
-from .common_util import StallMixin, run_cli
+from .common_util import StallMixin, run_cli_unicode
 from .common_web import do_http
 from allmydata.test.no_network import GridTestMixin
 from .cli.common import CLITestMixin
 
+
+def run_cli(verb, *argv):
+    """Match usage in existing tests by accept *args."""
+    return run_cli_unicode(verb, list(argv))
+
+
 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
     def test_good(self):
         self.basedir = "deepcheck/MutableChecker/good"
         self.set_up_grid()
-        CONTENTS = "a little bit of data"
+        CONTENTS = b"a little bit of data"
         CONTENTS_uploadable = MutableData(CONTENTS)
         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
         def _created(node):
             self.node = node
-            self.fileurl = "uri/" + urllib.quote(node.get_uri())
+            self.fileurl = "uri/" + url_quote(node.get_uri())
         d.addCallback(_created)
         # now make sure the webapi verifier sees no problems
         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
                                            method="POST"))
         def _got_results(out):
-            self.failUnless("<span>Healthy : Healthy</span>" in out, out)
+            self.failUnless(b"<span>Healthy : Healthy</span>" in out, out)
-            self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
+            self.failUnless(b"Recoverable Versions: 10*seq1-" in out, out)
-            self.failIf("Not Healthy!" in out, out)
+            self.failIf(b"Not Healthy!" in out, out)
-            self.failIf("Unhealthy" in out, out)
+            self.failIf(b"Unhealthy" in out, out)
-            self.failIf("Corrupt Shares" in out, out)
+            self.failIf(b"Corrupt Shares" in out, out)
         d.addCallback(_got_results)
         d.addErrback(self.explain_web_error)
         return d
@@ -61,12 +79,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
     def test_corrupt(self):
         self.basedir = "deepcheck/MutableChecker/corrupt"
         self.set_up_grid()
-        CONTENTS = "a little bit of data"
+        CONTENTS = b"a little bit of data"
         CONTENTS_uploadable = MutableData(CONTENTS)
         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
         def _stash_and_corrupt(node):
             self.node = node
-            self.fileurl = "uri/" + urllib.quote(node.get_uri())
+            self.fileurl = "uri/" + url_quote(node.get_uri())
             self.corrupt_shares_numbered(node.get_uri(), [0],
                                          _corrupt_mutable_share_data)
         d.addCallback(_stash_and_corrupt)
@@ -74,9 +92,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
                                            method="POST"))
         def _got_results(out):
-            self.failUnless("Not Healthy!" in out, out)
+            self.failUnless(b"Not Healthy!" in out, out)
-            self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
+            self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
-            self.failUnless("Corrupt Shares:" in out, out)
+            self.failUnless(b"Corrupt Shares:" in out, out)
         d.addCallback(_got_results)
 
         # now make sure the webapi repairer can fix it
@@ -84,13 +102,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
                       self.GET(self.fileurl+"?t=check&verify=true&repair=true",
                                method="POST"))
         def _got_repair_results(out):
-            self.failUnless("<div>Repair successful</div>" in out, out)
+            self.failUnless(b"<div>Repair successful</div>" in out, out)
         d.addCallback(_got_repair_results)
         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
                                            method="POST"))
         def _got_postrepair_results(out):
-            self.failIf("Not Healthy!" in out, out)
+            self.failIf(b"Not Healthy!" in out, out)
-            self.failUnless("Recoverable Versions: 10*seq" in out, out)
+            self.failUnless(b"Recoverable Versions: 10*seq" in out, out)
         d.addCallback(_got_postrepair_results)
         d.addErrback(self.explain_web_error)
 
@@ -99,21 +117,21 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
     def test_delete_share(self):
         self.basedir = "deepcheck/MutableChecker/delete_share"
         self.set_up_grid()
-        CONTENTS = "a little bit of data"
+        CONTENTS = b"a little bit of data"
         CONTENTS_uploadable = MutableData(CONTENTS)
         d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
         def _stash_and_delete(node):
             self.node = node
-            self.fileurl = "uri/" + urllib.quote(node.get_uri())
+            self.fileurl = "uri/" + url_quote(node.get_uri())
             self.delete_shares_numbered(node.get_uri(), [0])
         d.addCallback(_stash_and_delete)
         # now make sure the webapi checker notices it
         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
                                            method="POST"))
         def _got_results(out):
-            self.failUnless("Not Healthy!" in out, out)
+            self.failUnless(b"Not Healthy!" in out, out)
-            self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
+            self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
-            self.failIf("Corrupt Shares" in out, out)
+            self.failIf(b"Corrupt Shares" in out, out)
         d.addCallback(_got_results)
 
         # now make sure the webapi repairer can fix it
@@ -121,13 +139,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
                       self.GET(self.fileurl+"?t=check&verify=false&repair=true",
                                method="POST"))
         def _got_repair_results(out):
-            self.failUnless("Repair successful" in out)
+            self.failUnless(b"Repair successful" in out)
         d.addCallback(_got_repair_results)
         d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
                                            method="POST"))
         def _got_postrepair_results(out):
-            self.failIf("Not Healthy!" in out, out)
+            self.failIf(b"Not Healthy!" in out, out)
-            self.failUnless("Recoverable Versions: 10*seq" in out)
+            self.failUnless(b"Recoverable Versions: 10*seq" in out)
         d.addCallback(_got_postrepair_results)
         d.addErrback(self.explain_web_error)
 
@@ -152,7 +170,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
         return data
 
     def parse_streamed_json(self, s):
-        for unit in s.split("\n"):
+        for unit in s.split(b"\n"):
             if not unit:
                 # stream should end with a newline, so split returns ""
                 continue
@@ -165,14 +183,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
     @inlineCallbacks
     def web(self, n, method="GET", **kwargs):
         # returns (data, url)
-        url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
+        url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri())
-               + "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
+               + "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()]))
         data = yield do_http(method, url, browser_like_redirects=True)
         returnValue((data,url))
 
     @inlineCallbacks
     def wait_for_operation(self, ophandle):
-        url = self.client_baseurls[0] + "operations/" + ophandle
+        url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
         url += "?t=status&output=JSON"
         while True:
             body = yield do_http("get", url)
@@ -184,7 +202,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
 
     @inlineCallbacks
     def get_operation_results(self, ophandle, output=None):
-        url = self.client_baseurls[0] + "operations/" + ophandle
+        url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
         url += "?t=status"
         if output:
             url += "&output=" + output
@@ -220,36 +238,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
             self.root_uri = n.get_uri()
         d.addCallback(_created_root)
         d.addCallback(lambda ign:
-            c0.create_mutable_file(MutableData("mutable file contents")))
+            c0.create_mutable_file(MutableData(b"mutable file contents")))
         d.addCallback(lambda n: self.root.set_node(u"mutable", n))
         def _created_mutable(n):
             self.mutable = n
             self.mutable_uri = n.get_uri()
         d.addCallback(_created_mutable)
 
-        large = upload.Data("Lots of data\n" * 1000, None)
+        large = upload.Data(b"Lots of data\n" * 1000, None)
         d.addCallback(lambda ign: self.root.add_file(u"large", large))
         def _created_large(n):
             self.large = n
             self.large_uri = n.get_uri()
         d.addCallback(_created_large)
 
-        small = upload.Data("Small enough for a LIT", None)
+        small = upload.Data(b"Small enough for a LIT", None)
         d.addCallback(lambda ign: self.root.add_file(u"small", small))
         def _created_small(n):
             self.small = n
             self.small_uri = n.get_uri()
         d.addCallback(_created_small)
 
-        small2 = upload.Data("Small enough for a LIT too", None)
+        small2 = upload.Data(b"Small enough for a LIT too", None)
         d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
         def _created_small2(n):
             self.small2 = n
             self.small2_uri = n.get_uri()
         d.addCallback(_created_small2)
 
-        empty_litdir_uri = "URI:DIR2-LIT:"
+        empty_litdir_uri = b"URI:DIR2-LIT:"
-        tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
+        tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
 
         d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
         def _created_empty_lit_dir(n):
@@ -292,7 +310,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
                                  sorted(self.g.get_all_serverids()),
                                  where)
             all_serverids = set()
-            for (shareid, servers) in cr.get_sharemap().items():
+            for (shareid, servers) in list(cr.get_sharemap().items()):
                 all_serverids.update([s.get_serverid() for s in servers])
             self.failUnlessEqual(sorted(all_serverids),
                                  sorted(self.g.get_all_serverids()),
@@ -397,14 +415,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         mutable = [f for f in files
                    if f["cap"] is not None
                    and f["cap"].startswith("URI:SSK:")][0]
-        self.failUnlessEqual(mutable["cap"], self.mutable_uri)
+        self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri)
         self.failIfEqual(mutable["cap"], mutable["verifycap"])
         self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
         # for immutable file, verifycap==repaircap!=filecap
         large = [f for f in files
                    if f["cap"] is not None
                    and f["cap"].startswith("URI:CHK:")][0]
-        self.failUnlessEqual(large["cap"], self.large_uri)
+        self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri)
         self.failIfEqual(large["cap"], large["verifycap"])
         self.failUnlessEqual(large["verifycap"], large["repaircap"])
         self.check_stats_good(stats)
@@ -524,7 +542,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
     def json_check_is_healthy(self, data, n, where, incomplete=False):
 
         self.failUnlessEqual(data["storage-index"],
-                             base32.b2a(n.get_storage_index()), where)
+                             str(base32.b2a(n.get_storage_index()), "ascii"), where)
         self.failUnless("summary" in data, (where, data))
         self.failUnlessEqual(data["summary"].lower(), "healthy",
                              "%s: '%s'" % (where, data["summary"]))
@@ -550,7 +568,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
                                  where)
             self.failUnless("sharemap" in r, where)
             all_serverids = set()
-            for (shareid, serverids_s) in r["sharemap"].items():
+            for (shareid, serverids_s) in list(r["sharemap"].items()):
                 all_serverids.update(serverids_s)
             self.failUnlessEqual(sorted(all_serverids),
                                  sorted([idlib.nodeid_b2a(sid)
@@ -562,7 +580,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
     def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
         self.failUnlessEqual(data["storage-index"],
-                             base32.b2a(n.get_storage_index()), where)
+                             str(base32.b2a(n.get_storage_index()), "ascii"), where)
         self.failUnlessEqual(data["repair-attempted"], False, where)
         self.json_check_is_healthy(data["pre-repair-results"],
                                    n, where, incomplete)
@@ -571,7 +589,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
     def json_full_deepcheck_is_healthy(self, data, n, where):
         self.failUnlessEqual(data["root-storage-index"],
-                             base32.b2a(n.get_storage_index()), where)
+                             str(base32.b2a(n.get_storage_index()), "ascii"), where)
         self.failUnlessEqual(data["count-objects-checked"], 3, where)
         self.failUnlessEqual(data["count-objects-healthy"], 3, where)
         self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
@@ -582,7 +600,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
     def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
         self.failUnlessEqual(data["root-storage-index"],
-                             base32.b2a(n.get_storage_index()), where)
+                             str(base32.b2a(n.get_storage_index()), "ascii"), where)
         self.failUnlessEqual(data["count-objects-checked"], 3, where)
 
         self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
@@ -728,6 +746,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
     def do_test_cli_good(self, ignored):
         d = defer.succeed(None)
+        if PY3:  # TODO fixme once Python 3 CLI porting is done
+            return d
         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
         d.addCallback(lambda ign: self.do_cli_manifest_stream2())
         d.addCallback(lambda ign: self.do_cli_manifest_stream3())
@@ -738,7 +758,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def _check_manifest_storage_index(self, out):
-        lines = [l for l in out.split("\n") if l]
+        lines = [l for l in out.split(b"\n") if l]
         self.failUnlessEqual(len(lines), 3)
         self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
         self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
@@ -749,7 +769,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split("\n") if l]
+            lines = [l for l in out.split(b"\n") if l]
             self.failUnlessEqual(len(lines), 8)
             caps = {}
             for l in lines:
@@ -794,7 +814,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split("\n") if l]
+            lines = [l for l in out.split(b"\n") if l]
             self.failUnlessEqual(len(lines), 3)
             self.failUnless(self.root.get_verify_cap().to_string() in lines)
             self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
@@ -807,7 +827,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         def _check(args):
             (rc, out, err) = args
             self.failUnlessEqual(err, "")
-            lines = [l for l in out.split("\n") if l]
+            lines = [l for l in out.split(b"\n") if l]
             self.failUnlessEqual(len(lines), 3)
             self.failUnless(self.root.get_repair_cap().to_string() in lines)
             self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
@@ -819,7 +839,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         d = self.do_cli("stats", self.root_uri)
         def _check3(args):
             (rc, out, err) = args
-            lines = [l.strip() for l in out.split("\n") if l]
+            lines = [l.strip() for l in out.split(b"\n") if l]
             self.failUnless("count-immutable-files: 1" in lines)
             self.failUnless("count-mutable-files: 1" in lines)
             self.failUnless("count-literal-files: 3" in lines)
@@ -905,17 +925,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
         d.addCallback(self.create_mangled, "large-unrecoverable")
         d.addCallback(lambda ignored: c0.create_dirnode())
         d.addCallback(self._stash_node, "broken")
-        large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
+        large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None)
         d.addCallback(lambda ignored:
                       self.nodes["broken"].add_file(u"large1", large1))
         d.addCallback(lambda ignored:
                       self.nodes["broken"].create_subdirectory(u"subdir-good"))
-        large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
+        large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None)
         d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
         d.addCallback(lambda ignored:
                       self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
         d.addCallback(self._stash_node, "subdir-unrecoverable")
-        large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
+        large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None)
         d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
         d.addCallback(lambda ignored:
                       self._delete_most_shares(self.nodes["broken"]))
@@ -928,14 +948,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
     def create_mangled(self, ignored, name):
         nodetype, mangletype = name.split("-", 1)
         if nodetype == "mutable":
-            mutable_uploadable = MutableData("mutable file contents")
+            mutable_uploadable = MutableData(b"mutable file contents")
             d = self.g.clients[0].create_mutable_file(mutable_uploadable)
-            d.addCallback(lambda n: self.root.set_node(str(name), n))
+            d.addCallback(lambda n: self.root.set_node(str(name), n))  # TODO drop str() once strings are unicode
         elif nodetype == "large":
-            large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
+            large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None)
             d = self.root.add_file(str(name), large)
         elif nodetype == "small":
-            small = upload.Data("Small enough for a LIT", None)
+            small = upload.Data(b"Small enough for a LIT", None)
             d = self.root.add_file(str(name), small)
 
         d.addCallback(self._stash_node, name)
@@ -959,10 +979,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
     def _corrupt_some_shares(self, node):
         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
             if shnum in (0,1):
-                yield run_cli("debug", "corrupt-share", native_str(sharefile))
+                yield run_cli("debug", "corrupt-share", sharefile)
 
     def _delete_most_shares(self, node):
-        self.delete_shares_numbered(node.get_uri(), range(1,10))
+        self.delete_shares_numbered(node.get_uri(), list(range(1,10)))
 
 
     def check_is_healthy(self, cr, where):
@@ -1199,11 +1219,11 @@ class Large(DeepCheckBase, unittest.TestCase):
             self.subdir_node = subdir_node
             kids = {}
             for i in range(1, COUNT):
-                litcap = LiteralFileURI("%03d-data" % i).to_string()
+                litcap = LiteralFileURI(b"%03d-data" % i).to_string()
                 kids[u"%03d-small" % i] = (litcap, litcap)
             return subdir_node.set_children(kids)
         d.addCallback(_add_children)
-        up = upload.Data("large enough for CHK" * 100, "")
+        up = upload.Data(b"large enough for CHK" * 100, b"")
         d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
 
         def _start_deepcheck(ignored):

src/allmydata/util/_python3.py

@@ -29,6 +29,7 @@ PORTED_MODULES = [
     "allmydata._monkeypatch",
     "allmydata.blacklist",
     "allmydata.check_results",
+    "allmydata.client",
     "allmydata.codec",
     "allmydata.control",
     "allmydata.crypto",
@@ -88,6 +89,7 @@ PORTED_MODULES = [
     "allmydata.storage.server",
     "allmydata.storage.shares",
     "allmydata.test.no_network",
+    "allmydata.test.matchers",
     "allmydata.test.mutable.util",
     "allmydata.unknown",
     "allmydata.uri",
@@ -159,12 +161,18 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
     "allmydata.test.test_checker",
+    "allmydata.test.test_client",
     "allmydata.test.test_codec",
     "allmydata.test.test_common_util",
     "allmydata.test.test_configutil",
     "allmydata.test.test_connection_status",
     "allmydata.test.test_crawler",
     "allmydata.test.test_crypto",
+
+    # Only partially ported, CLI-using test code is disabled for now until CLI
+    # is ported.
+    "allmydata.test.test_deepcheck",
+
     "allmydata.test.test_deferredutil",
     "allmydata.test.test_dictutil",
     "allmydata.test.test_dirnode",
@@ -219,4 +227,5 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_util",
     "allmydata.test.web.test_web",
     "allmydata.test.web.test_webish",
+    "allmydata.test.test_windows",
 ]

src/allmydata/util/eliotutil.py

@@ -32,7 +32,7 @@ from six import ensure_text
 from sys import (
     stdout,
 )
-from functools import wraps
+from functools import wraps, partial
 from logging import (
     INFO,
     Handler,
@@ -66,6 +66,7 @@ from eliot.twisted import (
     DeferredContext,
     inline_callbacks,
 )
+from eliot.testing import capture_logging as eliot_capture_logging
 
 from twisted.python.usage import (
     UsageError,
@@ -326,3 +327,10 @@ def log_call_deferred(action_type):
                 return DeferredContext(d).addActionFinish()
         return logged_f
     return decorate_log_call_deferred
+
+# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON
+# encoder we want to use when validating messages.
+if PY2:
+    capture_logging = eliot_capture_logging
+else:
+    capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder)