Merge remote-tracking branch 'origin/3552.test_system-python-3' into 3565.web-tests-python-3-part-1

nix/tahoe-lafs.nix

@@ -23,17 +23,12 @@ python.pkgs.buildPythonPackage rec {
     # This list is over-zealous because it's more work to disable individual
     # tests with in a module.
 
-    # test_system is a lot of integration-style tests that do a lot of real
-    # networking between many processes.  They sometimes fail spuriously.
-    rm src/allmydata/test/test_system.py
-
     # Many of these tests don't properly skip when i2p or tor dependencies are
     # not supplied (and we are not supplying them).
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
     rm src/allmydata/test/test_client.py
-    rm src/allmydata/test/test_runner.py
   '';
 
 

src/allmydata/client.py

@@ -714,7 +714,7 @@ class _Client(node.Node, pollmixin.PollMixin):
     def get_long_nodeid(self):
         # this matches what IServer.get_longname() says about us elsewhere
         vk_string = ed25519.string_from_verifying_key(self._node_public_key)
-        return remove_prefix(vk_string, "pub-")
+        return remove_prefix(vk_string, b"pub-")
 
     def get_long_tubid(self):
         return idlib.nodeid_b2a(self.nodeid)
@@ -898,10 +898,6 @@ class _Client(node.Node, pollmixin.PollMixin):
         if helper_furl in ("None", ""):
             helper_furl = None
 
-        # FURLs need to be bytes:
-        if helper_furl is not None:
-            helper_furl = helper_furl.encode("utf-8")
-
         DEP = self.encoding_params
         DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
         DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))

src/allmydata/immutable/downloader/finder.py

@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_str
 
 import time
 now = time.time
@@ -98,7 +99,7 @@ class ShareFinder(object):
 
     # internal methods
     def loop(self):
-        pending_s = ",".join([rt.server.get_name()
+        pending_s = ",".join([ensure_str(rt.server.get_name())
                               for rt in self.pending_requests]) # sort?
         self.log(format="ShareFinder loop: running=%(running)s"
                  " hungry=%(hungry)s, pending=%(pending)s",

src/allmydata/immutable/upload.py

@@ -11,6 +11,7 @@ from future.utils import PY2, native_str
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long, unicode
+from six import ensure_str
 
 import os, time, weakref, itertools
 from zope.interface import implementer
@@ -1825,7 +1826,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
     def startService(self):
         service.MultiService.startService(self)
         if self._helper_furl:
-            self.parent.tub.connectTo(self._helper_furl,
+            self.parent.tub.connectTo(ensure_str(self._helper_furl),
                                       self._got_helper)
 
     def _got_helper(self, helper):

src/allmydata/introducer/client.py

@@ -11,7 +11,7 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long
 
-from six import ensure_text
+from six import ensure_text, ensure_str
 
 import time
 from zope.interface import implementer
@@ -39,8 +39,6 @@ class IntroducerClient(service.Service, Referenceable):
                  nickname, my_version, oldest_supported,
                  sequencer, cache_filepath):
         self._tub = tub
-        if isinstance(introducer_furl, str):
-            introducer_furl = introducer_furl.encode("utf-8")
         self.introducer_furl = introducer_furl
 
         assert isinstance(nickname, str)
@@ -96,7 +94,7 @@ class IntroducerClient(service.Service, Referenceable):
     def startService(self):
         service.Service.startService(self)
         self._introducer_error = None
-        rc = self._tub.connectTo(self.introducer_furl, self._got_introducer)
+        rc = self._tub.connectTo(ensure_str(self.introducer_furl), self._got_introducer)
         self._introducer_reconnector = rc
         def connect_failed(failure):
             self.log("Initial Introducer connection failed: perhaps it's down",

src/allmydata/test/common_util.py

@@ -1,5 +1,8 @@
 from __future__ import print_function
 
+from future.utils import PY2, native_str
+from future.builtins import str as future_str
+
 import os
 import time
 import signal
@@ -48,24 +51,23 @@ def _getvalue(io):
     return io.read()
 
 
-def run_cli_bytes(verb, *args, **kwargs):
+def run_cli_native(verb, *args, **kwargs):
     """
-    Run a Tahoe-LAFS CLI command specified as bytes.
+    Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode
+    (on Python 3); basically, it accepts a native string.
 
     Most code should prefer ``run_cli_unicode`` which deals with all the
-    necessary encoding considerations.  This helper still exists so that novel
+    necessary encoding considerations.
-    misconfigurations can be explicitly tested (for example, receiving UTF-8
-    bytes when the system encoding claims to be ASCII).
 
-    :param bytes verb: The command to run.  For example, ``b"create-node"``.
+    :param native_str verb: The command to run.  For example, ``b"create-node"``.
 
-    :param [bytes] args: The arguments to pass to the command.  For example,
+    :param [native_str] args: The arguments to pass to the command.  For example,
         ``(b"--hostname=localhost",)``.
 
-    :param [bytes] nodeargs: Extra arguments to pass to the Tahoe executable
+    :param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable
         before ``verb``.
 
-    :param bytes stdin: Text to pass to the command via stdin.
+    :param native_str stdin: Text to pass to the command via stdin.
 
     :param NoneType|str encoding: The name of an encoding which stdout and
         stderr will be configured to use.  ``None`` means stdout and stderr
@@ -75,8 +77,8 @@ def run_cli_bytes(verb, *args, **kwargs):
     nodeargs = kwargs.pop("nodeargs", [])
     encoding = kwargs.pop("encoding", None)
     precondition(
-        all(isinstance(arg, bytes) for arg in [verb] + nodeargs + list(args)),
+        all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)),
-        "arguments to run_cli must be bytes -- convert using unicode_to_argv",
+        "arguments to run_cli must be a native string -- convert using unicode_to_argv",
         verb=verb,
         args=args,
         nodeargs=nodeargs,
@@ -135,15 +137,19 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
     if nodeargs is None:
         nodeargs = []
     precondition(
-        all(isinstance(arg, unicode) for arg in [verb] + nodeargs + argv),
+        all(isinstance(arg, future_str) for arg in [verb] + nodeargs + argv),
         "arguments to run_cli_unicode must be unicode",
         verb=verb,
         nodeargs=nodeargs,
         argv=argv,
     )
     codec = encoding or "ascii"
+    if PY2:
         encode = lambda t: None if t is None else t.encode(codec)
-    d = run_cli_bytes(
+    else:
+        # On Python 3 command-line parsing expects Unicode!
+        encode = lambda t: t
+    d = run_cli_native(
         encode(verb),
         nodeargs=list(encode(arg) for arg in nodeargs),
         stdin=encode(stdin),
@@ -161,7 +167,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
     return d
 
 
-run_cli = run_cli_bytes
+run_cli = run_cli_native
 
 
 def parse_cli(*argv):

src/allmydata/test/common_web.py

@@ -47,12 +47,17 @@ class VerboseError(Error):
 
 @inlineCallbacks
 def do_http(method, url, **kwargs):
+    """
+    Run HTTP query, return Deferred of body as bytes.
+    """
     response = yield treq.request(method, url, persistent=False, **kwargs)
     body = yield treq.content(response)
     # TODO: replace this with response.fail_for_status when
     # https://github.com/twisted/treq/pull/159 has landed
     if 400 <= response.code < 600:
-        raise VerboseError(response.code, response=body)
+        raise VerboseError(
+            response.code, response="For request {} to {}, got: {}".format(
+                method, url, body))
     returnValue(body)
 
 

src/allmydata/test/test_system.py

@@ -1,7 +1,22 @@
+"""
+Ported to Python 3, partially: test_filesystem* will be done in a future round.
+"""
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2, PY3
+if PY2:
+    # Don't import bytes since it causes issues on (so far unported) modules on Python 2.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str  # noqa: F401
+
+from past.builtins import chr as byteschr, long
+from six import ensure_text, ensure_str
 
 import os, re, sys, time, json
 from functools import partial
+from unittest import skipIf
 
 from bs4 import BeautifulSoup
 
@@ -40,7 +55,7 @@ from .common import (
     TEST_RSA_KEY_SIZE,
     SameProcessStreamEndpointAssigner,
 )
-from .common_web import do_http, Error
+from .common_web import do_http as do_http_bytes, Error
 from .web.common import (
     assert_soup_has_tag_with_attributes
 )
@@ -48,12 +63,28 @@ from .web.common import (
 # TODO: move this to common or common_util
 from allmydata.test.test_runner import RunBinTahoeMixin
 from . import common_util as testutil
-from .common_util import run_cli
+from .common_util import run_cli_unicode
 from ..scripts.common import (
     write_introducer,
 )
 
-LARGE_DATA = """
+def run_cli(*args, **kwargs):
+    """
+    Backwards compatible version so we don't have to change all the tests.
+    """
+    nodeargs = [ensure_text(a) for a in kwargs.pop("nodeargs", [])]
+    kwargs["nodeargs"] = nodeargs
+    return run_cli_unicode(
+        ensure_text(args[0]), [ensure_text(a) for a in args[1:]], **kwargs)
+
+
+def do_http(*args, **kwargs):
+    """Wrapper for do_http() that returns Unicode."""
+    return do_http_bytes(*args, **kwargs).addCallback(
+        lambda b: str(b, "utf-8"))
+
+
+LARGE_DATA = b"""
 This is some data to publish to the remote grid.., which needs to be large
 enough to not fit inside a LIT uri.
 """
@@ -629,7 +660,7 @@ def _render_config(config):
     """
     Convert a ``dict`` of ``dict`` of ``bytes`` to an ini-format string.
     """
-    return "\n\n".join(list(
+    return u"\n\n".join(list(
         _render_config_section(k, v)
         for (k, v)
         in config.items()
@@ -640,7 +671,7 @@ def _render_config_section(heading, values):
     Convert a ``bytes`` heading and a ``dict`` of ``bytes`` to an ini-format
     section as ``bytes``.
     """
-    return "[{}]\n{}\n".format(
+    return u"[{}]\n{}\n".format(
         heading, _render_section_values(values)
     )
 
@@ -649,8 +680,8 @@ def _render_section_values(values):
     Convert a ``dict`` of ``bytes`` to the body of an ini-format section as
     ``bytes``.
     """
-    return "\n".join(list(
+    return u"\n".join(list(
-        "{} = {}".format(k, v)
+        u"{} = {}".format(k, v)
         for (k, v)
         in sorted(values.items())
     ))
@@ -753,7 +784,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
         self.helper_furl = helper_furl
         if self.numclients >= 4:
-            with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+') as f:
+            with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f:
                 f.write(
                     "[client]\n"
                     "helper.furl = {}\n".format(helper_furl)
@@ -796,8 +827,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
         def setconf(config, which, section, feature, value):
             if which in feature_matrix.get((section, feature), {which}):
-                if isinstance(value, unicode):
-                    value = value.encode("utf-8")
                 config.setdefault(section, {})[feature] = value
 
         setnode = partial(setconf, config, which, "node")
@@ -870,7 +899,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         config = "[client]\n"
         if helper_furl:
             config += "helper.furl = %s\n" % helper_furl
-        basedir.child("tahoe.cfg").setContent(config)
+        basedir.child("tahoe.cfg").setContent(config.encode("utf-8"))
         private = basedir.child("private")
         private.makedirs()
         write_introducer(
@@ -980,12 +1009,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
     def test_upload_and_download_convergent(self):
         self.basedir = "system/SystemTest/test_upload_and_download_convergent"
-        return self._test_upload_and_download(convergence="some convergence string")
+        return self._test_upload_and_download(convergence=b"some convergence string")
 
     def _test_upload_and_download(self, convergence):
         # we use 4000 bytes of data, which will result in about 400k written
         # to disk among all our simulated nodes
-        DATA = "Some data to upload\n" * 200
+        DATA = b"Some data to upload\n" * 200
         d = self.set_up_nodes()
         def _check_connections(res):
             for c in self.clients:
@@ -993,7 +1022,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 all_peerids = c.get_storage_broker().get_all_serverids()
                 self.failUnlessEqual(len(all_peerids), self.numclients)
                 sb = c.storage_broker
-                permuted_peers = sb.get_servers_for_psi("a")
+                permuted_peers = sb.get_servers_for_psi(b"a")
                 self.failUnlessEqual(len(permuted_peers), self.numclients)
         d.addCallback(_check_connections)
 
@@ -1016,7 +1045,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             theuri = results.get_uri()
             log.msg("upload finished: uri is %s" % (theuri,))
             self.uri = theuri
-            assert isinstance(self.uri, str), self.uri
+            assert isinstance(self.uri, bytes), self.uri
             self.cap = uri.from_string(self.uri)
             self.n = self.clients[1].create_node_from_uri(self.uri)
         d.addCallback(_upload_done)
@@ -1050,17 +1079,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             d.addCallback(lambda ign:
                           n.read(MemoryConsumer(), offset=1, size=4))
             def _read_portion_done(mc):
-                self.failUnlessEqual("".join(mc.chunks), DATA[1:1+4])
+                self.failUnlessEqual(b"".join(mc.chunks), DATA[1:1+4])
             d.addCallback(_read_portion_done)
             d.addCallback(lambda ign:
                           n.read(MemoryConsumer(), offset=2, size=None))
             def _read_tail_done(mc):
-                self.failUnlessEqual("".join(mc.chunks), DATA[2:])
+                self.failUnlessEqual(b"".join(mc.chunks), DATA[2:])
             d.addCallback(_read_tail_done)
             d.addCallback(lambda ign:
                           n.read(MemoryConsumer(), size=len(DATA)+1000))
             def _read_too_much(mc):
-                self.failUnlessEqual("".join(mc.chunks), DATA)
+                self.failUnlessEqual(b"".join(mc.chunks), DATA)
             d.addCallback(_read_too_much)
 
             return d
@@ -1110,7 +1139,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             return connected
         d.addCallback(lambda ign: self.poll(_has_helper))
 
-        HELPER_DATA = "Data that needs help to upload" * 1000
+        HELPER_DATA = b"Data that needs help to upload" * 1000
         def _upload_with_helper(res):
             u = upload.Data(HELPER_DATA, convergence=convergence)
             d = self.extra_node.upload(u)
@@ -1144,7 +1173,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(fireEventually)
 
         def _upload_resumable(res):
-            DATA = "Data that needs help to upload and gets interrupted" * 1000
+            DATA = b"Data that needs help to upload and gets interrupted" * 1000
             u1 = CountingDataUploadable(DATA, convergence=convergence)
             u2 = CountingDataUploadable(DATA, convergence=convergence)
 
@@ -1266,7 +1295,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             s = stats["stats"]
             self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1)
             c = stats["counters"]
-            self.failUnless("storage_server.allocate" in c)
+            # Probably this should be Unicode eventually? But we haven't ported
+            # stats code yet.
+            self.failUnless(b"storage_server.allocate" in c)
         d.addCallback(_grab_stats)
 
         return d
@@ -1287,7 +1318,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 assert pieces[-5].startswith("client")
                 client_num = int(pieces[-5][-1])
                 storage_index_s = pieces[-1]
-                storage_index = si_a2b(storage_index_s)
+                storage_index = si_a2b(storage_index_s.encode("ascii"))
                 for sharename in filenames:
                     shnum = int(sharename)
                     filename = os.path.join(dirpath, sharename)
@@ -1320,7 +1351,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         elif which == "signature":
             signature = self.flip_bit(signature)
         elif which == "share_hash_chain":
-            nodenum = share_hash_chain.keys()[0]
+            nodenum = list(share_hash_chain.keys())[0]
             share_hash_chain[nodenum] = self.flip_bit(share_hash_chain[nodenum])
         elif which == "block_hash_tree":
             block_hash_tree[-1] = self.flip_bit(block_hash_tree[-1])
@@ -1343,11 +1374,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
     def test_mutable(self):
         self.basedir = "system/SystemTest/test_mutable"
-        DATA = "initial contents go here."  # 25 bytes % 3 != 0
+        DATA = b"initial contents go here."  # 25 bytes % 3 != 0
         DATA_uploadable = MutableData(DATA)
-        NEWDATA = "new contents yay"
+        NEWDATA = b"new contents yay"
         NEWDATA_uploadable = MutableData(NEWDATA)
-        NEWERDATA = "this is getting old"
+        NEWERDATA = b"this is getting old"
         NEWERDATA_uploadable = MutableData(NEWERDATA)
 
         d = self.set_up_nodes()
@@ -1396,7 +1427,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 self.failUnless("  share_hash_chain: " in output)
                 self.failUnless("  block_hash_tree: 1 nodes\n" in output)
                 expected = ("  verify-cap: URI:SSK-Verifier:%s:" %
-                            base32.b2a(storage_index))
+                            str(base32.b2a(storage_index), "ascii"))
                 self.failUnless(expected in output)
             except unittest.FailTest:
                 print()
@@ -1475,7 +1506,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                            for (client_num, storage_index, filename, shnum)
                            in shares ])
             assert len(where) == 10 # this test is designed for 3-of-10
-            for shnum, filename in where.items():
+            for shnum, filename in list(where.items()):
                 # shares 7,8,9 are left alone. read will check
                 # (share_hash_chain, block_hash_tree, share_data). New
                 # seqnum+R pairs will trigger a check of (seqnum, R, IV,
@@ -1525,9 +1556,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         def _check_empty_file(res):
             # make sure we can create empty files, this usually screws up the
             # segsize math
-            d1 = self.clients[2].create_mutable_file(MutableData(""))
+            d1 = self.clients[2].create_mutable_file(MutableData(b""))
             d1.addCallback(lambda newnode: newnode.download_best_version())
-            d1.addCallback(lambda res: self.failUnlessEqual("", res))
+            d1.addCallback(lambda res: self.failUnlessEqual(b"", res))
             return d1
         d.addCallback(_check_empty_file)
 
@@ -1550,7 +1581,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return d
 
     def flip_bit(self, good):
-        return good[:-1] + chr(ord(good[-1]) ^ 0x01)
+        return good[:-1] + byteschr(ord(good[-1:]) ^ 0x01)
 
     def mangle_uri(self, gooduri):
         # change the key, which changes the storage index, which means we'll
@@ -1571,6 +1602,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
     # the key, which should cause the download to fail the post-download
     # plaintext_hash check.
 
+    @skipIf(PY3, "Python 3 web support hasn't happened yet.")
     def test_filesystem(self):
         self.basedir = "system/SystemTest/test_filesystem"
         self.data = LARGE_DATA
@@ -1632,7 +1664,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             d1.addCallback(self.log, "publish finished")
             def _stash_uri(filenode):
                 self.uri = filenode.get_uri()
-                assert isinstance(self.uri, str), (self.uri, filenode)
+                assert isinstance(self.uri, bytes), (self.uri, filenode)
             d1.addCallback(_stash_uri)
             return d1
         d.addCallback(_made_subdir1)
@@ -1650,7 +1682,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return res
 
     def _do_publish_private(self, res):
-        self.smalldata = "sssh, very secret stuff"
+        self.smalldata = b"sssh, very secret stuff"
         ut = upload.Data(self.smalldata, convergence=None)
         d = self.clients[0].create_dirnode()
         d.addCallback(self.log, "GOT private directory")
@@ -1737,7 +1769,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope"))
 
             d1.addCallback(self.log, "doing add_file(ro)")
-            ut = upload.Data("I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence="99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
+            ut = upload.Data(b"I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence=b"99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
             d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut))
 
             d1.addCallback(self.log, "doing get(ro)")
@@ -1801,7 +1833,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                             "largest-directory-children": 3,
                             "largest-immutable-file": 112,
                             }
-                for k,v in expected.iteritems():
+                for k,v in list(expected.items()):
                     self.failUnlessEqual(stats[k], v,
                                          "stats[%s] was %s, not %s" %
                                          (k, stats[k], v))
@@ -1850,33 +1882,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return do_http("get", self.webish_url + urlpath)
 
     def POST(self, urlpath, use_helper=False, **fields):
-        sepbase = "boogabooga"
+        sepbase = b"boogabooga"
-        sep = "--" + sepbase
+        sep = b"--" + sepbase
         form = []
         form.append(sep)
-        form.append('Content-Disposition: form-data; name="_charset"')
+        form.append(b'Content-Disposition: form-data; name="_charset"')
-        form.append('')
+        form.append(b'')
-        form.append('UTF-8')
+        form.append(b'UTF-8')
         form.append(sep)
-        for name, value in fields.iteritems():
+        for name, value in fields.items():
             if isinstance(value, tuple):
                 filename, value = value
-                form.append('Content-Disposition: form-data; name="%s"; '
+                form.append(b'Content-Disposition: form-data; name="%s"; '
-                            'filename="%s"' % (name, filename.encode("utf-8")))
+                            b'filename="%s"' % (name, filename.encode("utf-8")))
             else:
-                form.append('Content-Disposition: form-data; name="%s"' % name)
+                form.append(b'Content-Disposition: form-data; name="%s"' % name)
-            form.append('')
+            form.append(b'')
-            form.append(str(value))
+            form.append(b"%s" % (value,))
             form.append(sep)
-        form[-1] += "--"
+        form[-1] += b"--"
-        body = ""
+        body = b""
         headers = {}
         if fields:
-            body = "\r\n".join(form) + "\r\n"
+            body = b"\r\n".join(form) + b"\r\n"
-            headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
+            headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii")
         return self.POST2(urlpath, body, headers, use_helper)
 
-    def POST2(self, urlpath, body="", headers={}, use_helper=False):
+    def POST2(self, urlpath, body=b"", headers={}, use_helper=False):
         if use_helper:
             url = self.helper_webish_url + urlpath
         else:
@@ -1884,7 +1916,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return do_http("post", url, data=body, headers=headers)
 
     def _test_web(self, res):
-        public = "uri/" + self._root_directory_uri
+        public = "uri/" + str(self._root_directory_uri, "ascii")
         d = self.GET("")
         def _got_welcome(page):
             html = page.replace('\n', ' ')
@@ -1893,7 +1925,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                             "I didn't see the right '%s' message in:\n%s" % (connected_re, page))
             # nodeids/tubids don't have any regexp-special characters
             nodeid_re = r'<th>Node ID:</th>\s*<td title="TubID: %s">%s</td>' % (
-                self.clients[0].get_long_tubid(), self.clients[0].get_long_nodeid())
+                self.clients[0].get_long_tubid(), str(self.clients[0].get_long_nodeid(), "ascii"))
             self.failUnless(re.search(nodeid_re, html),
                             "I didn't see the right '%s' message in:\n%s" % (nodeid_re, page))
             self.failUnless("Helper: 0 active uploads" in page)
@@ -1954,7 +1986,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # upload a file with PUT
         d.addCallback(self.log, "about to try PUT")
         d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
-                                           "new.txt contents"))
+                                           b"new.txt contents"))
         d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
         d.addCallback(self.failUnlessEqual, "new.txt contents")
         # and again with something large enough to use multiple segments,
@@ -1965,23 +1997,23 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 c.encoding_params['happy'] = 1
         d.addCallback(_new_happy_semantics)
         d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
-                                           "big" * 500000)) # 1.5MB
+                                           b"big" * 500000)) # 1.5MB
         d.addCallback(lambda res: self.GET(public + "/subdir3/big.txt"))
         d.addCallback(lambda res: self.failUnlessEqual(len(res), 1500000))
 
         # can we replace files in place?
         d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
-                                           "NEWER contents"))
+                                           b"NEWER contents"))
         d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
         d.addCallback(self.failUnlessEqual, "NEWER contents")
 
         # test unlinked POST
-        d.addCallback(lambda res: self.POST("uri", t="upload",
+        d.addCallback(lambda res: self.POST("uri", t=b"upload",
-                                            file=("new.txt", "data" * 10000)))
+                                            file=("new.txt", b"data" * 10000)))
         # and again using the helper, which exercises different upload-status
         # display code
-        d.addCallback(lambda res: self.POST("uri", use_helper=True, t="upload",
+        d.addCallback(lambda res: self.POST("uri", use_helper=True, t=b"upload",
-                                            file=("foo.txt", "data2" * 10000)))
+                                            file=("foo.txt", b"data2" * 10000)))
 
         # check that the status page exists
         d.addCallback(lambda res: self.GET("status"))
@@ -2105,7 +2137,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # exercise some of the diagnostic tools in runner.py
 
         # find a share
-        for (dirpath, dirnames, filenames) in os.walk(unicode(self.basedir)):
+        for (dirpath, dirnames, filenames) in os.walk(ensure_text(self.basedir)):
             if "storage" not in dirpath:
                 continue
             if not filenames:
@@ -2119,7 +2151,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                 filename = os.path.join(dirpath, filenames[0])
                 # peek at the magic to see if it is a chk share
                 magic = open(filename, "rb").read(4)
-                if magic == '\x00\x00\x00\x01':
+                if magic == b'\x00\x00\x00\x01':
                     break
         else:
             self.fail("unable to find any uri_extension files in %r"
@@ -2152,7 +2184,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # 'find-shares' tool
         sharedir, shnum = os.path.split(filename)
         storagedir, storage_index_s = os.path.split(sharedir)
-        storage_index_s = str(storage_index_s)
         nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
         rc,out,err = yield run_cli("debug", "find-shares", storage_index_s,
                                    *nodedirs)
@@ -2176,7 +2207,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # allmydata.control (mostly used for performance tests)
         c0 = self.clients[0]
         control_furl_file = c0.config.get_private_path("control.furl")
-        control_furl = open(control_furl_file, "r").read().strip()
+        control_furl = ensure_str(open(control_furl_file, "r").read().strip())
         # it doesn't really matter which Tub we use to connect to the client,
         # so let's just use our IntroducerNode's
         d = self.introducer.tub.getReference(control_furl)
@@ -2208,7 +2239,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         # sure that works, before we add other aliases.
 
         root_file = os.path.join(client0_basedir, "private", "root_dir.cap")
-        f = open(root_file, "w")
+        f = open(root_file, "wb")
         f.write(private_uri)
         f.close()
 
@@ -2551,6 +2582,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         return d
 
+    @skipIf(PY3, "Python 3 CLI support hasn't happened yet.")
     def test_filesystem_with_cli_in_subprocess(self):
         # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
 
@@ -2574,12 +2606,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             out, err, rc_or_sig = res
             self.failUnlessEqual(rc_or_sig, 0, str(res))
             if check_stderr:
-                self.failUnlessEqual(err, "")
+                self.failUnlessEqual(err, b"")
 
         d.addCallback(_run_in_subprocess, "create-alias", "newalias")
         d.addCallback(_check_succeeded)
 
-        STDIN_DATA = "This is the file to upload from stdin."
+        STDIN_DATA = b"This is the file to upload from stdin."
         d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
         d.addCallback(_check_succeeded, check_stderr=False)
 
@@ -2601,7 +2633,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return d
 
     def _test_checker(self, res):
-        ut = upload.Data("too big to be literal" * 200, convergence=None)
+        ut = upload.Data(b"too big to be literal" * 200, convergence=None)
         d = self._personal_node.add_file(u"big file", ut)
 
         d.addCallback(lambda res: self._personal_node.check(Monitor()))

src/allmydata/test/test_util.py

@@ -33,7 +33,9 @@ if six.PY3:
 
 class IDLib(unittest.TestCase):
     def test_nodeid_b2a(self):
-        self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
+        result = idlib.nodeid_b2a(b"\x00"*20)
+        self.assertEqual(result, "a"*32)
+        self.assertIsInstance(result, str)
 
 
 class MyList(list):

src/allmydata/test/web/common.py

@@ -25,7 +25,8 @@ def assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs):
     tags = soup.find_all(tag_name)
     for tag in tags:
         if all(v in tag.attrs.get(k, []) for k, v in attrs.items()):
-            return  # we found every attr in this tag; done
+            # we found every attr in this tag; done
+            return tag
     testcase.fail(
         u"No <{}> tags contain attributes: {}".format(tag_name, attrs)
     )

src/allmydata/test/web/test_root.py

@@ -1,7 +1,13 @@
-from mock import Mock
-
 import time
 
+from urllib import (
+    quote,
+)
+
+from bs4 import (
+    BeautifulSoup,
+)
+
 from twisted.trial import unittest
 from twisted.web.template import Tag
 from twisted.web.test.requesthelper import DummyRequest
@@ -16,6 +22,9 @@ from ...util.connection_status import ConnectionStatus
 from allmydata.web.root import URIHandler
 from allmydata.client import _Client
 
+from .common import (
+    assert_soup_has_tag_with_attributes,
+)
 from ..common_web import (
     render,
 )
@@ -30,28 +39,37 @@ class RenderSlashUri(unittest.TestCase):
     """
 
     def setUp(self):
-        self.client = Mock()
+        self.client = object()
         self.res = URIHandler(self.client)
 
-    def test_valid(self):
+    def test_valid_query_redirect(self):
         """
-        A valid capbility does not result in error
+        A syntactically valid capability given in the ``uri`` query argument
+        results in a redirect.
         """
-        query_args = {b"uri": [
+        cap = (
             b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:"
             b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882"
-        ]}
+        )
+        query_args = {b"uri": [cap]}
         response_body = self.successResultOf(
             render(self.res, query_args),
         )
-        self.assertNotEqual(
+        soup = BeautifulSoup(response_body, 'html5lib')
-            response_body,
+        tag = assert_soup_has_tag_with_attributes(
-            "Invalid capability",
+            self,
+            soup,
+            u"meta",
+            {u"http-equiv": "refresh"},
+        )
+        self.assertIn(
+            quote(cap, safe=""),
+            tag.attrs.get(u"content"),
         )
 
     def test_invalid(self):
         """
-        A (trivially) invalid capbility is an error
+        A syntactically invalid capbility results in an error.
         """
         query_args = {b"uri": [b"not a capability"]}
         response_body = self.successResultOf(

src/allmydata/util/_python3.py

@@ -97,6 +97,7 @@ PORTED_MODULES = [
     "allmydata.util.happinessutil",
     "allmydata.util.hashutil",
     "allmydata.util.humanreadable",
+    "allmydata.util.idlib",
     "allmydata.util.iputil",
     "allmydata.util.jsonbytes",
     "allmydata.util.log",
@@ -168,6 +169,12 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_storage",
     "allmydata.test.test_storage_client",
     "allmydata.test.test_storage_web",
+
+    # Only partially ported, test_filesystem_with_cli_in_subprocess and
+    # test_filesystem methods aren't ported yet, should be done once CLI and
+    # web are ported respectively.
+    "allmydata.test.test_system",
+
     "allmydata.test.test_time_format",
     "allmydata.test.test_upload",
     "allmydata.test.test_uri",

src/allmydata/util/base32.py

@@ -133,6 +133,9 @@ def a2b(cs):
     """
     @param cs the base-32 encoded data (as bytes)
     """
+    # Workaround Future newbytes issues by converting to real bytes on Python 2:
+    if hasattr(cs, "__native__"):
+        cs = cs.__native__()
     precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
     precondition(isinstance(cs, bytes), cs)
 

src/allmydata/util/idlib.py

@@ -1,9 +1,29 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+from six import ensure_text
 from foolscap import base32
+
+
 def nodeid_b2a(nodeid):
-    # we display nodeids using the same base32 alphabet that Foolscap uses
+    """
-    return base32.encode(nodeid)
+    We display nodeids using the same base32 alphabet that Foolscap uses.
+
+    Returns a Unicode string.
+    """
+    return ensure_text(base32.encode(nodeid))
 
 def shortnodeid_b2a(nodeid):
+    """
+    Short version of nodeid_b2a() output, Unicode string.
+    """
     return nodeid_b2a(nodeid)[:8]