Merge pull request #1005 from tahoe-lafs/3634.cli-tests-python-3

Start porting CLI tests to Python 3

Fixes ticket:3634

src/allmydata/scripts/common.py

@@ -229,19 +229,19 @@ def get_alias(aliases, path_unicode, default):
     precondition(isinstance(path_unicode, str), path_unicode)
 
     from allmydata import uri
-    path = path_unicode.encode('utf-8').strip(" ")
+    path = path_unicode.encode('utf-8').strip(b" ")
     if uri.has_uri_prefix(path):
         # We used to require "URI:blah:./foo" in order to get a subpath,
         # stripping out the ":./" sequence. We still allow that for compatibility,
         # but now also allow just "URI:blah/foo".
-        sep = path.find(":./")
+        sep = path.find(b":./")
         if sep != -1:
             return path[:sep], path[sep+3:]
-        sep = path.find("/")
+        sep = path.find(b"/")
         if sep != -1:
             return path[:sep], path[sep+1:]
-        return path, ""
-    colon = path.find(":")
+        return path, b""
+    colon = path.find(b":")
     if colon == -1:
         # no alias
         if default == None:

src/allmydata/scripts/common_http.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 
 import os
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from six.moves import urllib, http_client
 import six
 import allmydata # for __full_version__
@@ -38,9 +38,9 @@ class BadResponse(object):
         return ""
 
 
-def do_http(method, url, body=""):
-    if isinstance(body, str):
-        body = StringIO(body)
+def do_http(method, url, body=b""):
+    if isinstance(body, bytes):
+        body = BytesIO(body)
     elif isinstance(body, six.text_type):
         raise TypeError("do_http body must be a bytestring, not unicode")
     else:

src/allmydata/scripts/tahoe_add_alias.py

@@ -1,9 +1,10 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
+from past.builtins import unicode
+
 import os.path
 import codecs
-import json
 
 from allmydata.util.assertutil import precondition
 
@@ -12,6 +13,7 @@ from allmydata.scripts.common_http import do_http, check_http_error
 from allmydata.scripts.common import get_aliases
 from allmydata.util.fileutil import move_into_place
 from allmydata.util.encodingutil import quote_output, quote_output_u
+from allmydata.util import jsonbytes as json
 
 
 def add_line_to_aliasfile(aliasfile, alias, cap):
@@ -52,7 +54,7 @@ def add_alias(options):
         show_output(stderr, "Alias {alias} already exists!", alias=alias)
         return 1
     aliasfile = os.path.join(nodedir, "private", "aliases")
-    cap = uri.from_string_dirnode(cap).to_string()
+    cap = unicode(uri.from_string_dirnode(cap).to_string(), 'utf-8')
 
     add_line_to_aliasfile(aliasfile, alias, cap)
     show_output(stdout, "Alias {alias} added", alias=alias)
@@ -92,7 +94,7 @@ def create_alias(options):
 
     # probably check for others..
 
-    add_line_to_aliasfile(aliasfile, alias, new_uri)
+    add_line_to_aliasfile(aliasfile, alias, unicode(new_uri, "utf-8"))
     show_output(stdout, "Alias {alias} created", alias=alias)
     return 0
 
@@ -167,7 +169,10 @@ def list_aliases(options):
     data = _get_alias_details(options['node-directory'])
 
     if options['json']:
-        output = _escape_format(json.dumps(data, indent=4).decode("ascii"))
+        dumped = json.dumps(data, indent=4)
+        if isinstance(dumped, bytes):
+            dumped = dumped.decode("utf-8")
+        output = _escape_format(dumped)
     else:
         def dircap(details):
             return (

src/allmydata/scripts/tahoe_status.py

@@ -1,7 +1,9 @@
 from __future__ import print_function
 
+from future.builtins import chr
+
 import os
-import urllib
+from urllib.parse import urlencode, quote as url_quote
 
 import json
 
@@ -25,12 +27,12 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None):
     if method == 'POST':
         if post_args is None:
             raise ValueError("Must pass post_args= for POST method")
-        body = urllib.urlencode(post_args)
+        body = urlencode(post_args)
     else:
         body = ''
         if post_args is not None:
             raise ValueError("post_args= only valid for POST method")
-    resp = do_http(method, url, body=body)
+    resp = do_http(method, url, body=body.encode("utf-8"))
     if isinstance(resp, BadResponse):
         # specifically NOT using format_http_error() here because the
         # URL is pretty sensitive (we're doing /uri/<key>).
@@ -48,7 +50,7 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None):
 def _get_json_for_cap(options, cap):
     return _get_json_for_fragment(
         options,
-        'uri/%s?t=json' % urllib.quote(cap),
+        'uri/%s?t=json' % url_quote(cap),
     )
 
 def pretty_progress(percent, size=10, ascii=False):
@@ -74,8 +76,8 @@ def pretty_progress(percent, size=10, ascii=False):
 
         # unicode 0x2581 -> 2589 are vertical bar chunks, like rainbarf uses
         # and following are narrow -> wider bars
-        part = unichr(0x258f - part) # for smooth bar
-        # part = unichr(0x2581 + part) # for neater-looking thing
+        part = chr(0x258f - part) # for smooth bar
+        # part = chr(0x2581 + part) # for neater-looking thing
 
     # hack for 100+ full so we don't print extra really-narrow/high bar
     if percent >= 100.0:

src/allmydata/test/cli/common.py

@@ -55,5 +55,5 @@ class CLITestMixin(ReallyEqualMixin):
         verb = ensure_str(verb)
         args = [ensure_str(arg) for arg in args]
         client_dir = ensure_str(self.get_clientdir(i=client_num))
-        nodeargs = [ b"--node-directory", client_dir ]
+        nodeargs = [ "--node-directory", client_dir ]
         return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)

src/allmydata/test/cli/test_alias.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import json
 
 from twisted.trial import unittest
@@ -59,7 +71,7 @@ class ListAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
         # the node filesystem state.
         aliases = get_aliases(self.get_clientdir())
         self.assertIn(alias, aliases)
-        self.assertTrue(aliases[alias].startswith(u"URI:DIR2:"))
+        self.assertTrue(aliases[alias].startswith(b"URI:DIR2:"))
 
         # And inspect the state via the user interface list-aliases command
         # too.

src/allmydata/test/cli/test_status.py

@@ -1,10 +1,21 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from six import ensure_text
+
 import os
 import mock
-import json
 import tempfile
-from six.moves import StringIO
+from io import BytesIO, StringIO
 from os.path import join
-from UserDict import UserDict
 
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -22,6 +33,7 @@ from allmydata.immutable.downloader.status import DownloadStatus
 from allmydata.mutable.publish import PublishStatus
 from allmydata.mutable.retrieve import RetrieveStatus
 from allmydata.mutable.servermap import UpdateStatus
+from allmydata.util import jsonbytes as json
 
 from ..no_network import GridTestMixin
 from ..common_web import do_http
@@ -60,9 +72,8 @@ class ProgressBar(unittest.TestCase):
         )
 
 
-class _FakeOptions(UserDict, object):
+class _FakeOptions(dict):
     def __init__(self):
-        super(_FakeOptions, self).__init__()
         self._tmp = tempfile.mkdtemp()
         os.mkdir(join(self._tmp, 'private'), 0o777)
         with open(join(self._tmp, 'private', 'api_auth_token'), 'w') as f:
@@ -86,7 +97,7 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase):
 
         # upload something
         c0 = self.g.clients[0]
-        data = MutableData("data" * 100)
+        data = MutableData(b"data" * 100)
         filenode = yield c0.create_mutable_file(data)
         self.uri = filenode.get_uri()
 
@@ -97,8 +108,8 @@ class Integration(GridTestMixin, CLITestMixin, unittest.TestCase):
         d = self.do_cli('status')# '--verbose')
 
         def _check(ign):
-            code, stdout, stdin = ign
-            self.assertEqual(code, 0)
+            code, stdout, stderr = ign
+            self.assertEqual(code, 0, stderr)
             self.assertTrue('Skipped 1' in stdout)
         d.addCallback(_check)
         return d
@@ -124,18 +135,18 @@ class CommandStatus(unittest.TestCase):
     @mock.patch('sys.stdout', StringIO())
     def test_no_operations(self, http):
         values = [
-            StringIO(json.dumps({
+            StringIO(ensure_text(json.dumps({
                 "active": [],
                 "recent": [],
-            })),
-            StringIO(json.dumps({
+            }))),
+            StringIO(ensure_text(json.dumps({
                 "counters": {
                     "bytes_downloaded": 0,
                 },
                 "stats": {
                     "node.uptime": 0,
                 }
-            })),
+            }))),
         ]
         http.side_effect = lambda *args, **kw: values.pop(0)
         do_status(self.options)
@@ -145,14 +156,14 @@ class CommandStatus(unittest.TestCase):
     def test_simple(self, http):
         recent_items = active_items = [
             UploadStatus(),
-            DownloadStatus("abcd", 12345),
+            DownloadStatus(b"abcd", 12345),
             PublishStatus(),
             RetrieveStatus(),
             UpdateStatus(),
             FakeStatus(),
         ]
         values = [
-            StringIO(json.dumps({
+            BytesIO(json.dumps({
                 "active": list(
                     marshal_json(item)
                     for item
@@ -163,15 +174,15 @@ class CommandStatus(unittest.TestCase):
                     for item
                     in recent_items
                 ),
-            })),
-            StringIO(json.dumps({
+            }).encode("utf-8")),
+            BytesIO(json.dumps({
                 "counters": {
                     "bytes_downloaded": 0,
                 },
                 "stats": {
                     "node.uptime": 0,
                 }
-            })),
+            }).encode("utf-8")),
         ]
         http.side_effect = lambda *args, **kw: values.pop(0)
         do_status(self.options)

src/allmydata/util/_python3.py

@@ -141,7 +141,9 @@ PORTED_MODULES = [
 ]
 
 PORTED_TEST_MODULES = [
+    "allmydata.test.cli.test_alias",
     "allmydata.test.cli.test_create",
+    "allmydata.test.cli.test_status",
 
     "allmydata.test.mutable.test_checker",
     "allmydata.test.mutable.test_datahandle",