Merge remote-tracking branch 'origin/master' into 3638.mypy-vs-allmydata-scripts

newsfragments/3629.feature

@@ -0,0 +1 @@
+The NixOS-packaged Tahoe-LAFS now knows its own version.

nix/tahoe-lafs.nix

@@ -1,5 +1,5 @@
 { fetchFromGitHub, lib
-, python
+, git, python
 , twisted, foolscap, zfec
 , setuptools, setuptoolsTrial, pyasn1, zope_interface
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
@@ -9,7 +9,35 @@
 python.pkgs.buildPythonPackage rec {
   version = "1.14.0.dev";
   name = "tahoe-lafs-${version}";
-  src = lib.cleanSource ../.;
+  src = lib.cleanSourceWith {
+    src = ../.;
+    filter = name: type:
+      let
+        basename = baseNameOf name;
+
+        split = lib.splitString ".";
+        join = builtins.concatStringsSep ".";
+        ext = join (builtins.tail (split basename));
+
+        # Build up a bunch of knowledge about what kind of file this is.
+        isTox = type == "directory" && basename == ".tox";
+        isTrialTemp = type == "directory" && basename == "_trial_temp";
+        isVersion = basename == "version.py";
+        isBytecode = ext == "pyc" || ext == "pyo";
+        isBackup = lib.hasSuffix "~" basename;
+        isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename;
+        isSymlink = type == "symlink";
+      in
+      # Exclude all these things
+      ! (isTrialTemp
+      || isTox
+      || isVersion
+      || isBytecode
+      || isBackup
+      || isTemporary
+      || isSymlink
+      );
+  };
 
   postPatch = ''
     # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around
@@ -24,13 +52,10 @@ python.pkgs.buildPythonPackage rec {
     # tests with in a module.
 
     # Many of these tests don't properly skip when i2p or tor dependencies are
-    # not supplied (and we are not supplying them). test_client.py fails because
+    # not supplied (and we are not supplying them).
-    # version is "unknown" on Nix.
-    # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter.
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
-    rm src/allmydata/test/test_client.py
 
     # Since we're deleting files, this complains they're missing. For now Nix
     # is Python 2-only, anyway, so these tests don't add anything yet.
@@ -38,6 +63,10 @@ python.pkgs.buildPythonPackage rec {
   '';
 
 
+  nativeBuildInputs = [
+    git
+  ];
+
   propagatedBuildInputs = with python.pkgs; [
     twisted foolscap zfec appdirs
     setuptoolsTrial pyasn1 zope_interface

src/allmydata/frontends/auth.py

@@ -7,6 +7,7 @@ from twisted.cred import error, checkers, credentials
 from twisted.conch.ssh import keys
 from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB
 
+from allmydata.util.dictutil import BytesKeyDict
 from allmydata.util import base32
 from allmydata.util.fileutil import abspath_expanduser_unicode
 
@@ -28,18 +29,18 @@ class AccountFileChecker(object):
                             credentials.ISSHPrivateKey)
     def __init__(self, client, accountfile):
         self.client = client
-        self.passwords = {}
+        self.passwords = BytesKeyDict()
-        pubkeys = {}
+        pubkeys = BytesKeyDict()
-        self.rootcaps = {}
+        self.rootcaps = BytesKeyDict()
-        with open(abspath_expanduser_unicode(accountfile), "r") as f:
+        with open(abspath_expanduser_unicode(accountfile), "rb") as f:
             for line in f:
                 line = line.strip()
-                if line.startswith("#") or not line:
+                if line.startswith(b"#") or not line:
                     continue
                 name, passwd, rest = line.split(None, 2)
-                if passwd.startswith("ssh-"):
+                if passwd.startswith(b"ssh-"):
                     bits = rest.split()
-                    keystring = " ".join([passwd] + bits[:-1])
+                    keystring = b" ".join([passwd] + bits[:-1])
                     key = keys.Key.fromString(keystring)
                     rootcap = bits[-1]
                     pubkeys[name] = [key]

src/allmydata/scripts/common.py

@@ -1,8 +1,12 @@
-from __future__ import print_function
+# coding: utf-8
 
-import os, sys, urllib, textwrap
+from __future__ import print_function
+from six import ensure_str
+
+import os, sys, textwrap
 import codecs
 from os.path import join
+import urllib.parse
 
 try:
     from typing import Optional
@@ -270,6 +274,18 @@ def get_alias(aliases, path_unicode, default):
     return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
 
 def escape_path(path):
-    # this always returns bytes, specifically US-ASCII, valid URL characters
+    # type: (str) -> str
+    u"""
+    Return path quoted to US-ASCII, valid URL characters.
+
+    >>> path = u'/føö/bar/☃'
+    >>> escaped = escape_path(path)
+    >>> str(escaped)
+    '/f%C3%B8%C3%B6/bar/%E2%98%83'
+    >>> escaped.encode('ascii').decode('ascii') == escaped
+    True
+    """
     segments = path.split("/")
-    return "/".join([urllib.quote(unicode_to_url(s)) for s in segments])
+    result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])
+    result = ensure_str(result, "ascii")
+    return result

src/allmydata/scripts/create_node.py

@@ -1,5 +1,15 @@
-from __future__ import print_function
+# Ported to Python 3
 
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+import io
 import os
 import json
 
@@ -229,7 +239,7 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions):
 @defer.inlineCallbacks
 def write_node_config(c, config):
     # this is shared between clients and introducers
-    c.write("# -*- mode: conf; coding: utf-8 -*-\n")
+    c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c))
     c.write("\n")
     c.write("# This file controls the configuration of the Tahoe node that\n")
     c.write("# lives in this directory. It is only read at node startup.\n")
@@ -248,7 +258,7 @@ def write_node_config(c, config):
 
     c.write("[node]\n")
     nickname = argv_to_unicode(config.get("nickname") or "")
-    c.write("nickname = %s\n" % (nickname.encode('utf-8'),))
+    c.write("nickname = %s\n" % (nickname,))
     if config["hide-ip"]:
         c.write("reveal-IP-address = false\n")
     else:
@@ -258,7 +268,7 @@ def write_node_config(c, config):
     webport = argv_to_unicode(config.get("webport") or "none")
     if webport.lower() == "none":
         webport = ""
-    c.write("web.port = %s\n" % (webport.encode('utf-8'),))
+    c.write("web.port = %s\n" % (webport,))
     c.write("web.static = public_html\n")
 
     listeners = config['listen'].split(",")
@@ -283,15 +293,14 @@ def write_node_config(c, config):
             tub_locations.append(i2p_location)
         if "tcp" in listeners:
             if config["port"]: # --port/--location are a pair
-                tub_ports.append(config["port"].encode('utf-8'))
+                tub_ports.append(config["port"])
-                tub_locations.append(config["location"].encode('utf-8'))
+                tub_locations.append(config["location"])
             else:
                 assert "hostname" in config
                 hostname = config["hostname"]
                 new_port = iputil.allocate_tcp_port()
                 tub_ports.append("tcp:%s" % new_port)
-                tub_locations.append("tcp:%s:%s" % (hostname.encode('utf-8'),
+                tub_locations.append("tcp:%s:%s" % (hostname, new_port))
-                                                    new_port))
         c.write("tub.port = %s\n" % ",".join(tub_ports))
         c.write("tub.location = %s\n" % ",".join(tub_locations))
     c.write("\n")
@@ -305,13 +314,13 @@ def write_node_config(c, config):
 
     if tor_config:
         c.write("[tor]\n")
-        for key, value in tor_config.items():
+        for key, value in list(tor_config.items()):
             c.write("%s = %s\n" % (key, value))
         c.write("\n")
 
     if i2p_config:
         c.write("[i2p]\n")
-        for key, value in i2p_config.items():
+        for key, value in list(i2p_config.items()):
             c.write("%s = %s\n" % (key, value))
         c.write("\n")
 
@@ -374,7 +383,7 @@ def _get_config_via_wormhole(config):
         relay_url=relay_url,
         reactor=reactor,
     )
-    code = unicode(config['join'])
+    code = str(config['join'])
     wh.set_code(code)
     yield wh.get_welcome()
     print("Connected to wormhole server", file=out)
@@ -406,7 +415,7 @@ def create_node(config):
     err = config.stderr
     basedir = config['basedir']
     # This should always be called with an absolute Unicode basedir.
-    precondition(isinstance(basedir, unicode), basedir)
+    precondition(isinstance(basedir, str), basedir)
 
     if os.path.exists(basedir):
         if listdir_unicode(basedir):
@@ -441,7 +450,7 @@ def create_node(config):
             v = remote_config.get(k, None)
             if v is not None:
                 # we're faking usually argv-supplied options :/
-                if isinstance(v, unicode):
+                if isinstance(v, str):
                     v = v.encode(get_io_encoding())
                 config[k] = v
                 if k not in sensitive_keys:
@@ -451,7 +460,8 @@ def create_node(config):
                     print("  {}: [sensitive data; see tahoe.cfg]".format(k), file=out)
 
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
-    with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
+    cfg_name = os.path.join(basedir, "tahoe.cfg")
+    with io.open(cfg_name, "w", encoding='utf-8') as c:
         yield write_node_config(c, config)
         write_client_config(c, config)
 
@@ -479,7 +489,7 @@ def create_introducer(config):
     err = config.stderr
     basedir = config['basedir']
     # This should always be called with an absolute Unicode basedir.
-    precondition(isinstance(basedir, unicode), basedir)
+    precondition(isinstance(basedir, str), basedir)
 
     if os.path.exists(basedir):
         if listdir_unicode(basedir):
@@ -493,7 +503,8 @@ def create_introducer(config):
     write_tac(basedir, "introducer")
 
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
-    with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
+    cfg_name = os.path.join(basedir, "tahoe.cfg")
+    with io.open(cfg_name, "w", encoding='utf-8') as c:
         yield write_node_config(c, config)
 
     print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)

src/allmydata/scripts/runner.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import warnings
 import os, sys
 from six.moves import StringIO
 import six
@@ -15,7 +16,7 @@ from twisted.internet import defer, task, threads
 from allmydata.scripts.common import get_default_nodedir
 from allmydata.scripts import debug, create_node, cli, \
     admin, tahoe_run, tahoe_invite
-from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding
+from allmydata.util.encodingutil import quote_local_unicode_path
 from allmydata.util.eliotutil import (
     opt_eliot_destination,
     opt_help_eliot_destinations,
@@ -120,11 +121,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout):
         while hasattr(c, 'subOptions'):
             c = c.subOptions
         print(str(c), file=stdout)
-        try:
+        print("%s:  %s\n" % (sys.argv[0], e), file=stdout)
-            msg = e.args[0].decode(get_io_encoding())
-        except Exception:
-            msg = repr(e)
-        print("%s:  %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False)), file=stdout)
         sys.exit(1)
     return config
 
@@ -177,9 +174,9 @@ def _maybe_enable_eliot_logging(options, reactor):
     return options
 
 def run():
-    # TODO(3035): Remove tox-check when error becomes a warning
+    if six.PY3:
-    if 'TOX_ENV_NAME' not in os.environ:
+        warnings.warn("Support for Python 3 is an incomplete work-in-progress."
-        assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x."
+                      " Use at your own risk.")
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize

src/allmydata/scripts/types_.py

@@ -1,3 +1,9 @@
+"""
+Type definitions used by modules in this package.
+"""
+
+# Python 3 only
+
 from typing import List, Tuple, Type, Sequence, Any
 from twisted.python.usage import Options
 

src/allmydata/test/cli/test_create.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os
 import mock
 from twisted.trial import unittest

src/allmydata/test/common.py

@@ -323,7 +323,7 @@ class AdoptedServerPort(object):
     """
     prefix = "adopt-socket"
 
-    def parseStreamServer(self, reactor, fd):
+    def parseStreamServer(self, reactor, fd): # type: ignore # https://twistedmatrix.com/trac/ticket/10134
         log.msg("Adopting {}".format(fd))
         # AdoptedStreamServerEndpoint wants to own the file descriptor.  It
         # will duplicate it and then close the one we pass in.  This means it
@@ -425,6 +425,9 @@ class DummyProducer(object):
     def resumeProducing(self):
         pass
 
+    def stopProducing(self):
+        pass
+
 @implementer(IImmutableFileNode)
 class FakeCHKFileNode(object):  # type: ignore # incomplete implementation
     """I provide IImmutableFileNode, but all of my data is stored in a

src/allmydata/test/common_util.py

@@ -1,6 +1,6 @@
 from __future__ import print_function
 
-from future.utils import PY2, native_str, bchr, binary_type
+from future.utils import PY2, bchr, binary_type
 from future.builtins import str as future_str
 from past.builtins import unicode
 
@@ -20,7 +20,7 @@ from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv
 
 
 def skip_if_cannot_represent_filename(u):
@@ -49,6 +49,13 @@ def _getvalue(io):
     return io.read()
 
 
+def maybe_unicode_to_argv(o):
+    """Convert object to argv form if necessary."""
+    if isinstance(o, unicode):
+        return unicode_to_argv(o)
+    return o
+
+
 def run_cli_native(verb, *args, **kwargs):
     """
     Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode
@@ -74,9 +81,12 @@ def run_cli_native(verb, *args, **kwargs):
     """
     nodeargs = kwargs.pop("nodeargs", [])
     encoding = kwargs.pop("encoding", None)
+    verb = maybe_unicode_to_argv(verb)
+    args = [maybe_unicode_to_argv(a) for a in args]
+    nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs]
     precondition(
-        all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)),
+        all(isinstance(arg, argv_type) for arg in [verb] + nodeargs + list(args)),
-        "arguments to run_cli must be a native string -- convert using unicode_to_argv",
+        "arguments to run_cli must be {argv_type} -- convert using unicode_to_argv".format(argv_type=argv_type),
         verb=verb,
         args=args,
         nodeargs=nodeargs,

src/allmydata/test/test_auth.py

@@ -1,4 +1,11 @@
-# Python 2 compatibility
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401
@@ -35,7 +42,7 @@ DUMMY_ACCOUNTS = u"""\
 alice password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111
 bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222
 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333
-""".format(key=DUMMY_KEY.public().toString("openssh")).encode("ascii")
+""".format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii")
 
 class AccountFileCheckerKeyTests(unittest.TestCase):
     """

src/allmydata/test/test_hung_server.py

@@ -1,5 +1,17 @@
 # -*- coding: utf-8 -*-
 
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os, shutil
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -14,8 +26,8 @@ from allmydata.test.common import ShouldFailMixin
 from allmydata.util.pollmixin import PollMixin
 from allmydata.interfaces import NotEnoughSharesError
 
-immutable_plaintext = "data" * 10000
+immutable_plaintext = b"data" * 10000
-mutable_plaintext = "muta" * 10000
+mutable_plaintext = b"muta" * 10000
 
 class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
                              unittest.TestCase):
@@ -105,7 +117,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
                 self.shares = self.find_uri_shares(self.uri)
             d.addCallback(_uploaded_mutable)
         else:
-            data = upload.Data(immutable_plaintext, convergence="")
+            data = upload.Data(immutable_plaintext, convergence=b"")
             d = self.c0.upload(data)
             def _uploaded_immutable(upload_res):
                 self.uri = upload_res.get_uri()
@@ -262,7 +274,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
             # is shut off. That will leave 4 OVERDUE and 1
             # stuck-but-not-overdue, for a total of 5 requests in in
             # _sf.pending_requests
-            for t in self._sf.overdue_timers.values()[:4]:
+            for t in list(self._sf.overdue_timers.values())[:4]:
                 t.reset(-1.0)
             # the timers ought to fire before the eventual-send does
             return fireEventually()

src/allmydata/test/test_multi_introducers.py

@@ -175,6 +175,3 @@ class NoDefault(unittest.TestCase):
         self.yaml_path.setContent(yamlutil.safe_dump(connections))
         myclient = yield create_client(self.basedir)
         self.assertEquals(len(myclient.introducer_clients), 0)
-
-if __name__ == "__main__":
-    unittest.main()

src/allmydata/test/test_repairer.py

@@ -30,9 +30,6 @@ MAX_DELTA_READS = 10 * READ_LEEWAY # N = 10
 timeout=240 # François's ARM box timed out after 120 seconds of Verifier.test_corrupt_crypttext_hashtree
 
 class RepairTestMixin(object):
-    def failUnlessIsInstance(self, x, xtype):
-        self.failUnless(isinstance(x, xtype), x)
-
     def _count_reads(self):
         sum_of_read_counts = 0
         for (i, ss, storedir) in self.iterate_servers():

src/allmydata/test/test_runner.py

@@ -1,7 +1,19 @@
+"""
+Ported to Python 3
+"""
 
 from __future__ import (
     absolute_import,
 )
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from six import ensure_text
 
 import os.path, re, sys
 from os import linesep
@@ -23,7 +35,7 @@ from twisted.python.runtime import (
     platform,
 )
 from allmydata.util import fileutil, pollmixin
-from allmydata.util.encodingutil import unicode_to_argv, get_filesystem_encoding
+from allmydata.util.encodingutil import unicode_to_argv
 from allmydata.test import common_util
 import allmydata
 from .common import (
@@ -72,7 +84,8 @@ def run_bintahoe(extra_argv, python_options=None):
     :return: A three-tuple of stdout (unicode), stderr (unicode), and the
         child process "returncode" (int).
     """
-    argv = [sys.executable.decode(get_filesystem_encoding())]
+    executable = ensure_text(sys.executable)
+    argv = [executable]
     if python_options is not None:
         argv.extend(python_options)
     argv.extend([u"-m", u"allmydata.scripts.runner"])
@@ -167,7 +180,7 @@ class CreateNode(unittest.TestCase):
 
         n1 = os.path.join(basedir, command + "-n1")
         argv = ["--quiet", command, "--basedir", n1] + list(args)
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -179,7 +192,7 @@ class CreateNode(unittest.TestCase):
             # 'create-node', and disabled for 'create-client'.
             tahoe_cfg = os.path.join(n1, "tahoe.cfg")
             self.failUnless(os.path.exists(tahoe_cfg))
-            content = fileutil.read(tahoe_cfg).replace('\r\n', '\n')
+            content = fileutil.read(tahoe_cfg).decode('utf-8').replace('\r\n', '\n')
             if kind == "client":
                 self.failUnless(re.search(r"\n\[storage\]\n#.*\nenabled = false\n", content), content)
             else:
@@ -187,7 +200,7 @@ class CreateNode(unittest.TestCase):
                 self.failUnless("\nreserved_space = 1G\n" in content)
 
         # creating the node a second time should be rejected
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failIfEqual(rc, 0, str((out, err, rc)))
         self.failUnlessEqual(out, "")
         self.failUnless("is not empty." in err)
@@ -200,7 +213,7 @@ class CreateNode(unittest.TestCase):
         # test that the non --basedir form works too
         n2 = os.path.join(basedir, command + "-n2")
         argv = ["--quiet", command] + list(args) + [n2]
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -210,7 +223,7 @@ class CreateNode(unittest.TestCase):
         # test the --node-directory form
         n3 = os.path.join(basedir, command + "-n3")
         argv = ["--quiet", "--node-directory", n3, command] + list(args)
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -221,7 +234,7 @@ class CreateNode(unittest.TestCase):
             # test that the output (without --quiet) includes the base directory
             n4 = os.path.join(basedir, command + "-n4")
             argv = [command] + list(args) + [n4]
-            rc, out, err = yield run_cli(*argv)
+            rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
             self.failUnlessEqual(err, "")
             self.failUnlessIn(" created in ", out)
             self.failUnlessIn(n4, out)
@@ -291,7 +304,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         # This makes sure that node.url is written, which allows us to
         # detect when the introducer restarts in _node_has_restarted below.
-        config = fileutil.read(tahoe.config_file.path)
+        config = fileutil.read(tahoe.config_file.path).decode('utf-8')
         self.assertIn('{}web.port = {}'.format(linesep, linesep), config)
         fileutil.write(
             tahoe.config_file.path,
@@ -303,7 +316,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         p = Expect()
         tahoe.run(on_stdout(p))
-        yield p.expect("introducer running")
+        yield p.expect(b"introducer running")
         tahoe.active()
 
         yield self.poll(tahoe.introducer_furl_file.exists)
@@ -326,7 +339,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         p = Expect()
         tahoe.run(on_stdout(p))
-        yield p.expect("introducer running")
+        yield p.expect(b"introducer running")
 
         # Again, the second incarnation of the node might not be ready yet, so
         # poll until it is. This time introducer_furl_file already exists, so
@@ -369,7 +382,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         self.failUnlessEqual(returncode, 0)
 
         # Check that the --webport option worked.
-        config = fileutil.read(tahoe.config_file.path)
+        config = fileutil.read(tahoe.config_file.path).decode('utf-8')
         self.assertIn(
             '{}web.port = 0{}'.format(linesep, linesep),
             config,
@@ -382,7 +395,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # This will run until we stop it.
         tahoe.run(on_stdout(p))
         # Wait for startup to have proceeded to a reasonable point.
-        yield p.expect("client running")
+        yield p.expect(b"client running")
         tahoe.active()
 
         # read the storage.furl file so we can check that its contents don't
@@ -401,7 +414,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # We don't have to add another cleanup for this one, the one from
         # above is still registered.
         tahoe.run(on_stdout(p))
-        yield p.expect("client running")
+        yield p.expect(b"client running")
         tahoe.active()
 
         self.assertEqual(
@@ -492,7 +505,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         client_running = p.expect(b"client running")
 
         result, index = yield DeferredList([
-            p.expect(expected_message),
+            p.expect(expected_message.encode('utf-8')),
             client_running,
         ], fireOnOneCallback=True, consumeErrors=True,
         )

src/allmydata/test/test_testing.py

@@ -9,7 +9,18 @@
 
 """
 Tests for the allmydata.testing helpers
+
+Ported to Python 3.
+
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.internet.defer import (
     inlineCallbacks,
@@ -79,7 +90,7 @@ class FakeWebTest(TestCase):
             self.assertThat(cap, IsInstance(CHKFileURI))
 
             resp = yield http_client.get(
-                "http://example.com/uri?uri={}".format(cap.to_string())
+                b"http://example.com/uri?uri=" + cap.to_string()
             )
             self.assertThat(resp.code, Equals(200))
 
@@ -88,7 +99,7 @@ class FakeWebTest(TestCase):
             # using the form "/uri/<cap>" is also valid
 
             resp = yield http_client.get(
-                "http://example.com/uri/{}".format(cap.to_string())
+                b"http://example.com/uri?uri=" + cap.to_string()
             )
             self.assertEqual(resp.code, 200)
 
@@ -136,9 +147,9 @@ class FakeWebTest(TestCase):
         """
 
         http_client = create_tahoe_treq_client()
-        cap_gen = capability_generator("URI:CHK:")
+        cap_gen = capability_generator(b"URI:CHK:")
-
+        cap = next(cap_gen).decode('ascii')
-        uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(next(cap_gen)))
+        uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(cap))
         resp = http_client.get(uri.to_uri().to_text())
 
         self.assertThat(

src/allmydata/test/web/test_web.py

@@ -108,7 +108,7 @@ class FakeNodeMaker(NodeMaker):
         return n.create(contents, version=version)
 
 class FakeUploader(service.Service):
-    name = "uploader"
+    name = "uploader" # type: ignore # https://twistedmatrix.com/trac/ticket/10135
     helper_furl = None
     helper_connected = False
 
@@ -254,7 +254,7 @@ class FakeLeaseChecker(object):
                 "remaining-wait-time": 0}
 
 class FakeStorageServer(service.MultiService):
-    name = 'storage'
+    name = 'storage' # type: ignore # https://twistedmatrix.com/trac/ticket/10135
     def __init__(self, nodeid, nickname):
         service.MultiService.__init__(self)
         self.my_nodeid = nodeid

src/allmydata/testing/web.py

@@ -6,10 +6,20 @@
 # This file is part of Tahoe-LAFS.
 #
 # See the docs/about.rst file for licensing information.
+"""Test-helpers for clients that use the WebUI.
+
+Ported to Python 3.
 
 """
-Test-helpers for clients that use the WebUI.
+from __future__ import absolute_import
-"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 import hashlib
 
@@ -84,16 +94,19 @@ def capability_generator(kind):
     given kind. The N, K and size values aren't related to anything
     real.
 
-    :param str kind: the kind of capability, like `URI:CHK`
+    :param bytes kind: the kind of capability, like `URI:CHK`
 
     :returns: a generator that yields new capablities of a particular
         kind.
     """
+    if not isinstance(kind, bytes):
+        raise TypeError("'kind' must be bytes")
+
     if kind not in KNOWN_CAPABILITIES:
         raise ValueError(
-            "Unknown capability kind '{} (valid are {})'".format(
+            "Unknown capability kind '{}' (valid are {})".format(
-                kind,
+                kind.decode('ascii'),
-                ", ".join(KNOWN_CAPABILITIES),
+                ", ".join([x.decode('ascii') for x in KNOWN_CAPABILITIES]),
             )
         )
     # what we do here is to start with empty hashers for the key and
@@ -108,16 +121,16 @@ def capability_generator(kind):
     # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size"
     while True:
         number += 1
-        key_hasher.update("\x00")
+        key_hasher.update(b"\x00")
-        ueb_hasher.update("\x00")
+        ueb_hasher.update(b"\x00")
 
         key = base32.b2a(key_hasher.digest()[:16])  # key is 16 bytes
         ueb_hash = base32.b2a(ueb_hasher.digest())  # ueb hash is 32 bytes
 
         cap = u"{kind}{key}:{ueb_hash}:{n}:{k}:{size}".format(
-            kind=kind,
+            kind=kind.decode('ascii'),
-            key=key,
+            key=key.decode('ascii'),
-            ueb_hash=ueb_hash,
+            ueb_hash=ueb_hash.decode('ascii'),
             n=1,
             k=1,
             size=number * 1000,
@@ -154,6 +167,8 @@ class _FakeTahoeUriHandler(Resource, object):
 
         :returns: a two-tuple: a bool (True if the data is freshly added) and a capability-string
         """
+        if not isinstance(kind, bytes):
+            raise TypeError("'kind' must be bytes")
         if not isinstance(data, bytes):
             raise TypeError("'data' must be bytes")
 
@@ -171,7 +186,7 @@ class _FakeTahoeUriHandler(Resource, object):
 
     def render_PUT(self, request):
         data = request.content.read()
-        fresh, cap = self.add_data("URI:CHK:", data)
+        fresh, cap = self.add_data(b"URI:CHK:", data)
         if fresh:
             request.setResponseCode(http.CREATED)  # real code does this for brand-new files
         else:
@@ -183,7 +198,7 @@ class _FakeTahoeUriHandler(Resource, object):
         data = request.content.read()
 
         type_to_kind = {
-            "mkdir-immutable": "URI:DIR2-CHK:"
+            "mkdir-immutable": b"URI:DIR2-CHK:"
         }
         kind = type_to_kind[t]
         fresh, cap = self.add_data(kind, data)
@@ -206,9 +221,10 @@ class _FakeTahoeUriHandler(Resource, object):
 
         # the user gave us a capability; if our Grid doesn't have any
         # data for it, that's an error.
+        capability = capability.encode('ascii')
         if capability not in self.data:
             request.setResponseCode(http.BAD_REQUEST)
-            return u"No data for '{}'".format(capability).decode("ascii")
+            return u"No data for '{}'".format(capability.decode('ascii'))
 
         return self.data[capability]
 

src/allmydata/util/_python3.py

@@ -76,6 +76,8 @@ PORTED_MODULES = [
     "allmydata.mutable.servermap",
     "allmydata.node",
     "allmydata.nodemaker",
+    "allmydata.scripts.create_node",
+    "allmydata.scripts.types_",
     "allmydata.stats",
     "allmydata.storage_client",
     "allmydata.storage.common",
@@ -89,6 +91,7 @@ PORTED_MODULES = [
     "allmydata.test.no_network",
     "allmydata.test.matchers",
     "allmydata.test.mutable.util",
+    "allmydata.testing.web",
     "allmydata.unknown",
     "allmydata.uri",
     "allmydata.util._python3",
@@ -138,6 +141,8 @@ PORTED_MODULES = [
 ]
 
 PORTED_TEST_MODULES = [
+    "allmydata.test.cli.test_create",
+
     "allmydata.test.mutable.test_checker",
     "allmydata.test.mutable.test_datahandle",
     "allmydata.test.mutable.test_different_encoding",
@@ -154,6 +159,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.mutable.test_update",
     "allmydata.test.mutable.test_version",
     "allmydata.test.test_abbreviate",
+    "allmydata.test.test_auth",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
     "allmydata.test.test_checker",
@@ -182,6 +188,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_hashutil",
     "allmydata.test.test_helper",
     "allmydata.test.test_humanreadable",
+    "allmydata.test.test_hung_server",
     "allmydata.test.test_immutable",
     "allmydata.test.test_introducer",
     "allmydata.test.test_iputil",
@@ -195,6 +202,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_pipeline",
     "allmydata.test.test_python3",
     "allmydata.test.test_repairer",
+    "allmydata.test.test_runner",
     "allmydata.test.test_sftp",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",
@@ -208,6 +216,7 @@ PORTED_TEST_MODULES = [
     # should be done once CLI is ported.
     "allmydata.test.test_system",
 
+    "allmydata.test.test_testing",
     "allmydata.test.test_time_format",
     "allmydata.test.test_upload",
     "allmydata.test.test_uri",

src/allmydata/util/encodingutil.py

@@ -13,6 +13,7 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 from future.utils import PY2, PY3, native_str
+from future.builtins import str as future_str
 if PY2:
     # We omit str() because that seems too tricky to get right.
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
@@ -142,6 +143,14 @@ def unicode_to_argv(s, mangle=False):
     return ensure_str(s)
 
 
+# According to unicode_to_argv above, the expected type for
+# cli args depends on the platform, so capture that expectation.
+argv_type = (future_str, native_str) if sys.platform == "win32" else native_str
+"""
+The expected type for args to a subprocess
+"""
+
+
 def unicode_to_url(s):
     """
     Encode an unicode object used in an URL to bytes.