diff --git a/newsfragments/3603.minor.rst b/newsfragments/3603.minor.rst new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index d73344274..36330f535 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -1,8 +1,12 @@ -from __future__ import print_function +# coding: utf-8 -import os, sys, urllib, textwrap +from __future__ import print_function +from six import ensure_str + +import os, sys, textwrap import codecs from os.path import join +import urllib.parse try: from typing import Optional @@ -270,6 +274,18 @@ def get_alias(aliases, path_unicode, default): return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:] def escape_path(path): - # this always returns bytes, specifically US-ASCII, valid URL characters + # type: (str) -> str + u""" + Return path quoted to US-ASCII, valid URL characters. + + >>> path = u'/føö/bar/☃' + >>> escaped = escape_path(path) + >>> str(escaped) + '/f%C3%B8%C3%B6/bar/%E2%98%83' + >>> escaped.encode('ascii').decode('ascii') == escaped + True + """ segments = path.split("/") - return "/".join([urllib.quote(unicode_to_url(s)) for s in segments]) + result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments]) + result = ensure_str(result, "ascii") + return result diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 0f507f518..3afbeeb0d 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -1,5 +1,15 @@ -from __future__ import print_function +# Ported to Python 3 +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import io import os import json @@ -225,7 +235,7 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions): @defer.inlineCallbacks def write_node_config(c, config): # this is shared between clients and introducers - c.write("# -*- mode: conf; coding: utf-8 -*-\n") + c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c)) c.write("\n") c.write("# This file controls the configuration of the Tahoe node that\n") c.write("# lives in this directory. It is only read at node startup.\n") @@ -244,7 +254,7 @@ def write_node_config(c, config): c.write("[node]\n") nickname = argv_to_unicode(config.get("nickname") or "") - c.write("nickname = %s\n" % (nickname.encode('utf-8'),)) + c.write("nickname = %s\n" % (nickname,)) if config["hide-ip"]: c.write("reveal-IP-address = false\n") else: @@ -254,7 +264,7 @@ def write_node_config(c, config): webport = argv_to_unicode(config.get("webport") or "none") if webport.lower() == "none": webport = "" - c.write("web.port = %s\n" % (webport.encode('utf-8'),)) + c.write("web.port = %s\n" % (webport,)) c.write("web.static = public_html\n") listeners = config['listen'].split(",") @@ -279,15 +289,14 @@ def write_node_config(c, config): tub_locations.append(i2p_location) if "tcp" in listeners: if config["port"]: # --port/--location are a pair - tub_ports.append(config["port"].encode('utf-8')) - tub_locations.append(config["location"].encode('utf-8')) + tub_ports.append(config["port"]) + tub_locations.append(config["location"]) else: assert "hostname" in config hostname = config["hostname"] new_port = iputil.allocate_tcp_port() tub_ports.append("tcp:%s" % new_port) - tub_locations.append("tcp:%s:%s" % (hostname.encode('utf-8'), - new_port)) + tub_locations.append("tcp:%s:%s" % (hostname, new_port)) c.write("tub.port = %s\n" % ",".join(tub_ports)) c.write("tub.location = %s\n" % ",".join(tub_locations)) c.write("\n") @@ -301,13 +310,13 @@ def write_node_config(c, config): if tor_config: c.write("[tor]\n") - for key, value in tor_config.items(): + for key, value in list(tor_config.items()): c.write("%s = %s\n" % (key, value)) c.write("\n") if i2p_config: c.write("[i2p]\n") - for key, value in i2p_config.items(): + for key, value in list(i2p_config.items()): c.write("%s = %s\n" % (key, value)) c.write("\n") @@ -370,7 +379,7 @@ def _get_config_via_wormhole(config): relay_url=relay_url, reactor=reactor, ) - code = unicode(config['join']) + code = str(config['join']) wh.set_code(code) yield wh.get_welcome() print("Connected to wormhole server", file=out) @@ -402,7 +411,7 @@ def create_node(config): err = config.stderr basedir = config['basedir'] # This should always be called with an absolute Unicode basedir. - precondition(isinstance(basedir, unicode), basedir) + precondition(isinstance(basedir, str), basedir) if os.path.exists(basedir): if listdir_unicode(basedir): @@ -437,7 +446,7 @@ def create_node(config): v = remote_config.get(k, None) if v is not None: # we're faking usually argv-supplied options :/ - if isinstance(v, unicode): + if isinstance(v, str): v = v.encode(get_io_encoding()) config[k] = v if k not in sensitive_keys: @@ -447,7 +456,8 @@ def create_node(config): print(" {}: [sensitive data; see tahoe.cfg]".format(k), file=out) fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) - with open(os.path.join(basedir, "tahoe.cfg"), "w") as c: + cfg_name = os.path.join(basedir, "tahoe.cfg") + with io.open(cfg_name, "w", encoding='utf-8') as c: yield write_node_config(c, config) write_client_config(c, config) @@ -475,7 +485,7 @@ def create_introducer(config): err = config.stderr basedir = config['basedir'] # This should always be called with an absolute Unicode basedir. - precondition(isinstance(basedir, unicode), basedir) + precondition(isinstance(basedir, str), basedir) if os.path.exists(basedir): if listdir_unicode(basedir): @@ -489,7 +499,8 @@ def create_introducer(config): write_tac(basedir, "introducer") fileutil.make_dirs(os.path.join(basedir, "private"), 0o700) - with open(os.path.join(basedir, "tahoe.cfg"), "w") as c: + cfg_name = os.path.join(basedir, "tahoe.cfg") + with io.open(cfg_name, "w", encoding='utf-8') as c: yield write_node_config(c, config) print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 9a632a57d..ada3c2dfc 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -1,5 +1,6 @@ from __future__ import print_function +import warnings import os, sys from six.moves import StringIO import six @@ -15,7 +16,7 @@ from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir from allmydata.scripts import debug, create_node, cli, \ admin, tahoe_run, tahoe_invite -from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding +from allmydata.util.encodingutil import quote_local_unicode_path from allmydata.util.eliotutil import ( opt_eliot_destination, opt_help_eliot_destinations, @@ -120,11 +121,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout): while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - try: - msg = e.args[0].decode(get_io_encoding()) - except Exception: - msg = repr(e) - print("%s: %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False)), file=stdout) + print("%s: %s\n" % (sys.argv[0], e), file=stdout) sys.exit(1) return config @@ -177,9 +174,9 @@ def _maybe_enable_eliot_logging(options, reactor): return options def run(): - # TODO(3035): Remove tox-check when error becomes a warning - if 'TOX_ENV_NAME' not in os.environ: - assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x." + if six.PY3: + warnings.warn("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.") if sys.platform == "win32": from allmydata.windows.fixups import initialize diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 3937cb803..289e674ce 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -1,3 +1,9 @@ +""" +Type definitions used by modules in this package. +""" + +# Python 3 only + from typing import List, Tuple, Type, Sequence, Any from allmydata.scripts.common import BaseOptions diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index aee07a671..282f26163 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import os import mock from twisted.trial import unittest diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index f898e75b2..f62cd34cc 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,6 +1,6 @@ from __future__ import print_function -from future.utils import PY2, native_str, bchr, binary_type +from future.utils import PY2, bchr, binary_type from future.builtins import str as future_str from past.builtins import unicode @@ -20,7 +20,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv def skip_if_cannot_represent_filename(u): @@ -49,6 +49,13 @@ def _getvalue(io): return io.read() +def maybe_unicode_to_argv(o): + """Convert object to argv form if necessary.""" + if isinstance(o, unicode): + return unicode_to_argv(o) + return o + + def run_cli_native(verb, *args, **kwargs): """ Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode @@ -74,9 +81,12 @@ def run_cli_native(verb, *args, **kwargs): """ nodeargs = kwargs.pop("nodeargs", []) encoding = kwargs.pop("encoding", None) + verb = maybe_unicode_to_argv(verb) + args = [maybe_unicode_to_argv(a) for a in args] + nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs] precondition( - all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)), - "arguments to run_cli must be a native string -- convert using unicode_to_argv", + all(isinstance(arg, argv_type) for arg in [verb] + nodeargs + list(args)), + "arguments to run_cli must be {argv_type} -- convert using unicode_to_argv".format(argv_type=argv_type), verb=verb, args=args, nodeargs=nodeargs, diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index cf56e8baa..f6a7c2ee1 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -1,7 +1,19 @@ +""" +Ported to Python 3 +""" from __future__ import ( absolute_import, ) +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from six import ensure_text import os.path, re, sys from os import linesep @@ -23,7 +35,7 @@ from twisted.python.runtime import ( platform, ) from allmydata.util import fileutil, pollmixin -from allmydata.util.encodingutil import unicode_to_argv, get_filesystem_encoding +from allmydata.util.encodingutil import unicode_to_argv from allmydata.test import common_util import allmydata from .common import ( @@ -72,7 +84,8 @@ def run_bintahoe(extra_argv, python_options=None): :return: A three-tuple of stdout (unicode), stderr (unicode), and the child process "returncode" (int). """ - argv = [sys.executable.decode(get_filesystem_encoding())] + executable = ensure_text(sys.executable) + argv = [executable] if python_options is not None: argv.extend(python_options) argv.extend([u"-m", u"allmydata.scripts.runner"]) @@ -167,7 +180,7 @@ class CreateNode(unittest.TestCase): n1 = os.path.join(basedir, command + "-n1") argv = ["--quiet", command, "--basedir", n1] + list(args) - rc, out, err = yield run_cli(*argv) + rc, out, err = yield run_cli(*map(unicode_to_argv, argv)) self.failUnlessEqual(err, "") self.failUnlessEqual(out, "") self.failUnlessEqual(rc, 0) @@ -179,7 +192,7 @@ class CreateNode(unittest.TestCase): # 'create-node', and disabled for 'create-client'. tahoe_cfg = os.path.join(n1, "tahoe.cfg") self.failUnless(os.path.exists(tahoe_cfg)) - content = fileutil.read(tahoe_cfg).replace('\r\n', '\n') + content = fileutil.read(tahoe_cfg).decode('utf-8').replace('\r\n', '\n') if kind == "client": self.failUnless(re.search(r"\n\[storage\]\n#.*\nenabled = false\n", content), content) else: @@ -187,7 +200,7 @@ class CreateNode(unittest.TestCase): self.failUnless("\nreserved_space = 1G\n" in content) # creating the node a second time should be rejected - rc, out, err = yield run_cli(*argv) + rc, out, err = yield run_cli(*map(unicode_to_argv, argv)) self.failIfEqual(rc, 0, str((out, err, rc))) self.failUnlessEqual(out, "") self.failUnless("is not empty." in err) @@ -200,7 +213,7 @@ class CreateNode(unittest.TestCase): # test that the non --basedir form works too n2 = os.path.join(basedir, command + "-n2") argv = ["--quiet", command] + list(args) + [n2] - rc, out, err = yield run_cli(*argv) + rc, out, err = yield run_cli(*map(unicode_to_argv, argv)) self.failUnlessEqual(err, "") self.failUnlessEqual(out, "") self.failUnlessEqual(rc, 0) @@ -210,7 +223,7 @@ class CreateNode(unittest.TestCase): # test the --node-directory form n3 = os.path.join(basedir, command + "-n3") argv = ["--quiet", "--node-directory", n3, command] + list(args) - rc, out, err = yield run_cli(*argv) + rc, out, err = yield run_cli(*map(unicode_to_argv, argv)) self.failUnlessEqual(err, "") self.failUnlessEqual(out, "") self.failUnlessEqual(rc, 0) @@ -221,7 +234,7 @@ class CreateNode(unittest.TestCase): # test that the output (without --quiet) includes the base directory n4 = os.path.join(basedir, command + "-n4") argv = [command] + list(args) + [n4] - rc, out, err = yield run_cli(*argv) + rc, out, err = yield run_cli(*map(unicode_to_argv, argv)) self.failUnlessEqual(err, "") self.failUnlessIn(" created in ", out) self.failUnlessIn(n4, out) @@ -291,7 +304,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # This makes sure that node.url is written, which allows us to # detect when the introducer restarts in _node_has_restarted below. - config = fileutil.read(tahoe.config_file.path) + config = fileutil.read(tahoe.config_file.path).decode('utf-8') self.assertIn('{}web.port = {}'.format(linesep, linesep), config) fileutil.write( tahoe.config_file.path, @@ -303,7 +316,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): p = Expect() tahoe.run(on_stdout(p)) - yield p.expect("introducer running") + yield p.expect(b"introducer running") tahoe.active() yield self.poll(tahoe.introducer_furl_file.exists) @@ -326,7 +339,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): p = Expect() tahoe.run(on_stdout(p)) - yield p.expect("introducer running") + yield p.expect(b"introducer running") # Again, the second incarnation of the node might not be ready yet, so # poll until it is. This time introducer_furl_file already exists, so @@ -369,7 +382,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(returncode, 0) # Check that the --webport option worked. - config = fileutil.read(tahoe.config_file.path) + config = fileutil.read(tahoe.config_file.path).decode('utf-8') self.assertIn( '{}web.port = 0{}'.format(linesep, linesep), config, @@ -382,7 +395,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # This will run until we stop it. tahoe.run(on_stdout(p)) # Wait for startup to have proceeded to a reasonable point. - yield p.expect("client running") + yield p.expect(b"client running") tahoe.active() # read the storage.furl file so we can check that its contents don't @@ -401,7 +414,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): # We don't have to add another cleanup for this one, the one from # above is still registered. tahoe.run(on_stdout(p)) - yield p.expect("client running") + yield p.expect(b"client running") tahoe.active() self.assertEqual( @@ -492,7 +505,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): client_running = p.expect(b"client running") result, index = yield DeferredList([ - p.expect(expected_message), + p.expect(expected_message.encode('utf-8')), client_running, ], fireOnOneCallback=True, consumeErrors=True, ) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bae105d9a..ec428fbe9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -76,6 +76,8 @@ PORTED_MODULES = [ "allmydata.mutable.servermap", "allmydata.node", "allmydata.nodemaker", + "allmydata.scripts.create_node", + "allmydata.scripts.types_", "allmydata.stats", "allmydata.storage_client", "allmydata.storage.common", @@ -138,6 +140,8 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.cli.test_create", + "allmydata.test.mutable.test_checker", "allmydata.test.mutable.test_datahandle", "allmydata.test.mutable.test_different_encoding", @@ -197,6 +201,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_pipeline", "allmydata.test.test_python3", "allmydata.test.test_repairer", + "allmydata.test.test_runner", "allmydata.test.test_sftp", "allmydata.test.test_spans", "allmydata.test.test_statistics", diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 483871b5d..637374064 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -13,6 +13,7 @@ from __future__ import print_function from __future__ import unicode_literals from future.utils import PY2, PY3, native_str +from future.builtins import str as future_str if PY2: # We omit str() because that seems too tricky to get right. from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 @@ -142,6 +143,14 @@ def unicode_to_argv(s, mangle=False): return ensure_str(s) +# According to unicode_to_argv above, the expected type for +# cli args depends on the platform, so capture that expectation. +argv_type = (future_str, native_str) if sys.platform == "win32" else native_str +""" +The expected type for args to a subprocess +""" + + def unicode_to_url(s): """ Encode an unicode object used in an URL to bytes.