diff --git a/newsfragments/3634.minor b/newsfragments/3634.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 36330f535..29342ec6b 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -229,19 +229,19 @@ def get_alias(aliases, path_unicode, default): precondition(isinstance(path_unicode, str), path_unicode) from allmydata import uri - path = path_unicode.encode('utf-8').strip(" ") + path = path_unicode.encode('utf-8').strip(b" ") if uri.has_uri_prefix(path): # We used to require "URI:blah:./foo" in order to get a subpath, # stripping out the ":./" sequence. We still allow that for compatibility, # but now also allow just "URI:blah/foo". - sep = path.find(":./") + sep = path.find(b":./") if sep != -1: return path[:sep], path[sep+3:] - sep = path.find("/") + sep = path.find(b"/") if sep != -1: return path[:sep], path[sep+1:] - return path, "" - colon = path.find(":") + return path, b"" + colon = path.find(b":") if colon == -1: # no alias if default == None: diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py index 5d989feb1..a53f6baa8 100644 --- a/src/allmydata/scripts/common_http.py +++ b/src/allmydata/scripts/common_http.py @@ -1,7 +1,7 @@ from __future__ import print_function import os -from six.moves import cStringIO as StringIO +from io import BytesIO from six.moves import urllib, http_client import six import allmydata # for __full_version__ @@ -38,9 +38,9 @@ class BadResponse(object): return "" -def do_http(method, url, body=""): - if isinstance(body, str): - body = StringIO(body) +def do_http(method, url, body=b""): + if isinstance(body, bytes): + body = BytesIO(body) elif isinstance(body, six.text_type): raise TypeError("do_http body must be a bytestring, not unicode") else: diff --git a/src/allmydata/scripts/tahoe_add_alias.py b/src/allmydata/scripts/tahoe_add_alias.py index 6f931556d..19474b9e8 100644 --- a/src/allmydata/scripts/tahoe_add_alias.py +++ b/src/allmydata/scripts/tahoe_add_alias.py @@ -1,9 +1,10 @@ from __future__ import print_function from __future__ import unicode_literals +from past.builtins import unicode + import os.path import codecs -import json from allmydata.util.assertutil import precondition @@ -12,6 +13,7 @@ from allmydata.scripts.common_http import do_http, check_http_error from allmydata.scripts.common import get_aliases from allmydata.util.fileutil import move_into_place from allmydata.util.encodingutil import quote_output, quote_output_u +from allmydata.util import jsonbytes as json def add_line_to_aliasfile(aliasfile, alias, cap): @@ -52,7 +54,7 @@ def add_alias(options): show_output(stderr, "Alias {alias} already exists!", alias=alias) return 1 aliasfile = os.path.join(nodedir, "private", "aliases") - cap = uri.from_string_dirnode(cap).to_string() + cap = unicode(uri.from_string_dirnode(cap).to_string(), 'utf-8') add_line_to_aliasfile(aliasfile, alias, cap) show_output(stdout, "Alias {alias} added", alias=alias) @@ -92,7 +94,7 @@ def create_alias(options): # probably check for others.. - add_line_to_aliasfile(aliasfile, alias, new_uri) + add_line_to_aliasfile(aliasfile, alias, unicode(new_uri, "utf-8")) show_output(stdout, "Alias {alias} created", alias=alias) return 0 @@ -167,7 +169,10 @@ def list_aliases(options): data = _get_alias_details(options['node-directory']) if options['json']: - output = _escape_format(json.dumps(data, indent=4).decode("ascii")) + dumped = json.dumps(data, indent=4) + if isinstance(dumped, bytes): + dumped = dumped.decode("utf-8") + output = _escape_format(dumped) else: def dircap(details): return ( diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 405a8c730..ff746901b 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -1,7 +1,9 @@ from __future__ import print_function +from future.builtins import chr + import os -import urllib +from urllib.parse import urlencode, quote as url_quote import json @@ -25,12 +27,12 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None): if method == 'POST': if post_args is None: raise ValueError("Must pass post_args= for POST method") - body = urllib.urlencode(post_args) + body = urlencode(post_args) else: body = '' if post_args is not None: raise ValueError("post_args= only valid for POST method") - resp = do_http(method, url, body=body) + resp = do_http(method, url, body=body.encode("utf-8")) if isinstance(resp, BadResponse): # specifically NOT using format_http_error() here because the # URL is pretty sensitive (we're doing /uri/). @@ -48,7 +50,7 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None): def _get_json_for_cap(options, cap): return _get_json_for_fragment( options, - 'uri/%s?t=json' % urllib.quote(cap), + 'uri/%s?t=json' % url_quote(cap), ) def pretty_progress(percent, size=10, ascii=False): @@ -74,8 +76,8 @@ def pretty_progress(percent, size=10, ascii=False): # unicode 0x2581 -> 2589 are vertical bar chunks, like rainbarf uses # and following are narrow -> wider bars - part = unichr(0x258f - part) # for smooth bar - # part = unichr(0x2581 + part) # for neater-looking thing + part = chr(0x258f - part) # for smooth bar + # part = chr(0x2581 + part) # for neater-looking thing # hack for 100+ full so we don't print extra really-narrow/high bar if percent >= 100.0: diff --git a/src/allmydata/test/cli/common.py b/src/allmydata/test/cli/common.py index 3033aed4b..8796f815f 100644 --- a/src/allmydata/test/cli/common.py +++ b/src/allmydata/test/cli/common.py @@ -55,5 +55,5 @@ class CLITestMixin(ReallyEqualMixin): verb = ensure_str(verb) args = [ensure_str(arg) for arg in args] client_dir = ensure_str(self.get_clientdir(i=client_num)) - nodeargs = [ b"--node-directory", client_dir ] + nodeargs = [ "--node-directory", client_dir ] return run_cli(verb, *args, nodeargs=nodeargs, **kwargs) diff --git a/src/allmydata/test/cli/test_alias.py b/src/allmydata/test/cli/test_alias.py index 07f42b29d..a3ee595b8 100644 --- a/src/allmydata/test/cli/test_alias.py +++ b/src/allmydata/test/cli/test_alias.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import json from twisted.trial import unittest @@ -59,7 +71,7 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase): # the node filesystem state. aliases = get_aliases(self.get_clientdir()) self.assertIn(alias, aliases) - self.assertTrue(aliases[alias].startswith(u"URI:DIR2:")) + self.assertTrue(aliases[alias].startswith(b"URI:DIR2:")) # And inspect the state via the user interface list-aliases command # too. diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 551b1a3e0..a04939429 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -1,10 +1,21 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_text + import os import mock -import json import tempfile -from six.moves import StringIO +from io import BytesIO, StringIO from os.path import join -from UserDict import UserDict from twisted.trial import unittest from twisted.internet import defer @@ -22,6 +33,7 @@ from allmydata.immutable.downloader.status import DownloadStatus from allmydata.mutable.publish import PublishStatus from allmydata.mutable.retrieve import RetrieveStatus from allmydata.mutable.servermap import UpdateStatus +from allmydata.util import jsonbytes as json from ..no_network import GridTestMixin from ..common_web import do_http @@ -60,9 +72,8 @@ class ProgressBar(unittest.TestCase): ) -class _FakeOptions(UserDict, object): +class _FakeOptions(dict): def __init__(self): - super(_FakeOptions, self).__init__() self._tmp = tempfile.mkdtemp() os.mkdir(join(self._tmp, 'private'), 0o777) with open(join(self._tmp, 'private', 'api_auth_token'), 'w') as f: @@ -86,7 +97,7 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): # upload something c0 = self.g.clients[0] - data = MutableData("data" * 100) + data = MutableData(b"data" * 100) filenode = yield c0.create_mutable_file(data) self.uri = filenode.get_uri() @@ -97,8 +108,8 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase): d = self.do_cli('status')# '--verbose') def _check(ign): - code, stdout, stdin = ign - self.assertEqual(code, 0) + code, stdout, stderr = ign + self.assertEqual(code, 0, stderr) self.assertTrue('Skipped 1' in stdout) d.addCallback(_check) return d @@ -124,18 +135,18 @@ class CommandStatus(unittest.TestCase): @mock.patch('sys.stdout', StringIO()) def test_no_operations(self, http): values = [ - StringIO(json.dumps({ + StringIO(ensure_text(json.dumps({ "active": [], "recent": [], - })), - StringIO(json.dumps({ + }))), + StringIO(ensure_text(json.dumps({ "counters": { "bytes_downloaded": 0, }, "stats": { "node.uptime": 0, } - })), + }))), ] http.side_effect = lambda *args, **kw: values.pop(0) do_status(self.options) @@ -145,14 +156,14 @@ class CommandStatus(unittest.TestCase): def test_simple(self, http): recent_items = active_items = [ UploadStatus(), - DownloadStatus("abcd", 12345), + DownloadStatus(b"abcd", 12345), PublishStatus(), RetrieveStatus(), UpdateStatus(), FakeStatus(), ] values = [ - StringIO(json.dumps({ + BytesIO(json.dumps({ "active": list( marshal_json(item) for item @@ -163,15 +174,15 @@ class CommandStatus(unittest.TestCase): for item in recent_items ), - })), - StringIO(json.dumps({ + }).encode("utf-8")), + BytesIO(json.dumps({ "counters": { "bytes_downloaded": 0, }, "stats": { "node.uptime": 0, } - })), + }).encode("utf-8")), ] http.side_effect = lambda *args, **kw: values.pop(0) do_status(self.options) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 14ca60e0a..35ec0410a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -141,7 +141,9 @@ PORTED_MODULES = [ ] PORTED_TEST_MODULES = [ + "allmydata.test.cli.test_alias", "allmydata.test.cli.test_create", + "allmydata.test.cli.test_status", "allmydata.test.mutable.test_checker", "allmydata.test.mutable.test_datahandle",