Merge pull request #1050 from tahoe-lafs/3687.cli-tests-python-3

Port more CLI tests to Python 3

Fixes ticket:3687
Fixes ticket:3679
This commit is contained in:
Itamar Turner-Trauring 2021-05-03 10:47:59 -04:00 committed by GitHub
commit 94b42b7c89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 228 additions and 158 deletions

0
newsfragments/3679.minor Normal file
View File

0
newsfragments/3687.minor Normal file
View File

View File

@ -224,7 +224,7 @@ class CpOptions(FileStoreOptions):
def parseArgs(self, *args):
if len(args) < 2:
raise usage.UsageError("cp requires at least two arguments")
self.sources = list(map(argv_to_unicode, args[:-1]))
self.sources = [argv_to_unicode(arg) for arg in args[:-1]]
self.destination = argv_to_unicode(args[-1])
synopsis = "[options] FROM.. TO"

View File

@ -1,8 +1,9 @@
from __future__ import print_function
from past.builtins import unicode
import os.path
import urllib
import json
from urllib.parse import quote as url_quote
from collections import defaultdict
from six.moves import cStringIO as StringIO
from twisted.python.failure import Failure
@ -15,6 +16,7 @@ from allmydata.util.fileutil import abspath_expanduser_unicode, precondition_abs
from allmydata.util.encodingutil import unicode_to_url, listdir_unicode, quote_output, \
quote_local_unicode_path, to_bytes
from allmydata.util.assertutil import precondition, _assert
from allmydata.util import jsonbytes as json
class MissingSourceError(TahoeError):
@ -61,8 +63,8 @@ def mkdir(targeturl):
def make_tahoe_subdirectory(nodeurl, parent_writecap, name):
url = nodeurl + "/".join(["uri",
urllib.quote(parent_writecap),
urllib.quote(unicode_to_url(name)),
url_quote(parent_writecap),
url_quote(unicode_to_url(name)),
]) + "?t=mkdir"
resp = do_http("POST", url)
if resp.status in (200, 201):
@ -199,7 +201,7 @@ class TahoeFileSource(object):
def open(self, caps_only):
if caps_only:
return StringIO(self.readcap)
url = self.nodeurl + "uri/" + urllib.quote(self.readcap)
url = self.nodeurl + "uri/" + url_quote(self.readcap)
return GET_to_file(url)
def bestcap(self):
@ -239,7 +241,7 @@ class TahoeDirectorySource(object):
self.writecap = writecap
self.readcap = readcap
bestcap = writecap or readcap
url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
url = self.nodeurl + "uri/%s" % url_quote(bestcap)
resp = do_http("GET", url + "?t=json")
if resp.status != 200:
raise HTTPError("Error examining source directory", resp)
@ -249,7 +251,7 @@ class TahoeDirectorySource(object):
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict( [(unicode(name),value)
for (name,value)
in d["children"].iteritems()] )
in d["children"].items()] )
self.children = None
def init_from_parsed(self, parsed):
@ -259,7 +261,7 @@ class TahoeDirectorySource(object):
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict( [(unicode(name),value)
for (name,value)
in d["children"].iteritems()] )
in d["children"].items()] )
self.children = None
def populate(self, recurse):
@ -329,14 +331,14 @@ class TahoeDirectoryTarget(object):
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict( [(unicode(name),value)
for (name,value)
in d["children"].iteritems()] )
in d["children"].items()] )
self.children = None
def init_from_grid(self, writecap, readcap):
self.writecap = writecap
self.readcap = readcap
bestcap = writecap or readcap
url = self.nodeurl + "uri/%s" % urllib.quote(bestcap)
url = self.nodeurl + "uri/%s" % url_quote(bestcap)
resp = do_http("GET", url + "?t=json")
if resp.status != 200:
raise HTTPError("Error examining target directory", resp)
@ -346,7 +348,7 @@ class TahoeDirectoryTarget(object):
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict( [(unicode(name),value)
for (name,value)
in d["children"].iteritems()] )
in d["children"].items()] )
self.children = None
def just_created(self, writecap):
@ -370,8 +372,8 @@ class TahoeDirectoryTarget(object):
url = None
if self.writecap:
url = self.nodeurl + "/".join(["uri",
urllib.quote(self.writecap),
urllib.quote(unicode_to_url(name))])
url_quote(self.writecap),
url_quote(unicode_to_url(name))])
self.children[name] = TahoeFileTarget(self.nodeurl, mutable,
writecap, readcap, url)
elif data[0] == "dirnode":
@ -439,7 +441,7 @@ class TahoeDirectoryTarget(object):
def set_children(self):
if not self.new_children:
return
url = (self.nodeurl + "uri/" + urllib.quote(self.writecap)
url = (self.nodeurl + "uri/" + url_quote(self.writecap)
+ "?t=set_children")
set_data = {}
for (name, filecap) in self.new_children.items():
@ -450,7 +452,7 @@ class TahoeDirectoryTarget(object):
# TODO: think about how this affects forward-compatibility for
# unknown caps
set_data[name] = ["filenode", {"rw_uri": filecap}]
body = json.dumps(set_data)
body = json.dumps_bytes(set_data)
POST(url, body)
FileSources = (LocalFileSource, TahoeFileSource)
@ -603,7 +605,7 @@ class Copier(object):
t = LocalFileTarget(pathname) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
if path:
url += "/" + escape_path(path)
@ -656,7 +658,7 @@ class Copier(object):
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
name = None
if path:
if path.endswith("/"):

View File

@ -45,10 +45,10 @@ def list(options):
return resp.status
data = resp.read()
if options['json']:
# The webapi server should always output printable ASCII.
if is_printable_ascii(data):
data = unicode(data, "ascii")
print(data, file=stdout)
return 0
else:

View File

@ -1,6 +1,8 @@
from __future__ import print_function
import urllib
from past.builtins import unicode
from urllib.parse import quote as url_quote
from allmydata.scripts.common_http import do_http, check_http_error
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, UnknownAliasError
from allmydata.util.encodingutil import quote_output
@ -24,7 +26,7 @@ def mkdir(options):
# create a new unlinked directory
url = nodeurl + "uri?t=mkdir"
if options["format"]:
url += "&format=%s" % urllib.quote(options['format'])
url += "&format=%s" % url_quote(options['format'])
resp = do_http("POST", url)
rc = check_http_error(resp, stderr)
if rc:
@ -35,13 +37,14 @@ def mkdir(options):
return 0
# create a new directory at the given location
path = unicode(path, "utf-8")
if path.endswith("/"):
path = path[:-1]
# path must be "/".join([s.encode("utf-8") for s in segments])
url = nodeurl + "uri/%s/%s?t=mkdir" % (urllib.quote(rootcap),
urllib.quote(path))
url = nodeurl + "uri/%s/%s?t=mkdir" % (url_quote(rootcap),
url_quote(path))
if options['format']:
url += "&format=%s" % urllib.quote(options['format'])
url += "&format=%s" % url_quote(options['format'])
resp = do_http("POST", url)
check_http_error(resp, stderr)

View File

@ -1,7 +1,7 @@
from __future__ import print_function
import re
import urllib
from urllib.parse import quote as url_quote
import json
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
UnknownAliasError
@ -25,7 +25,7 @@ def mv(options, mode="move"):
except UnknownAliasError as e:
e.display(stderr)
return 1
from_url = nodeurl + "uri/%s" % urllib.quote(rootcap)
from_url = nodeurl + "uri/%s" % url_quote(rootcap)
if from_path:
from_url += "/" + escape_path(from_path)
# figure out the source cap
@ -43,7 +43,7 @@ def mv(options, mode="move"):
except UnknownAliasError as e:
e.display(stderr)
return 1
to_url = nodeurl + "uri/%s" % urllib.quote(rootcap)
to_url = nodeurl + "uri/%s" % url_quote(rootcap)
if path:
to_url += "/" + escape_path(path)

View File

@ -1,7 +1,10 @@
from __future__ import print_function
from six.moves import cStringIO as StringIO
import urllib
from future.utils import PY2
from past.builtins import unicode
from io import BytesIO
from urllib.parse import quote as url_quote
from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
@ -46,19 +49,20 @@ def put(options):
# FIXME: don't hardcode cap format.
if to_file.startswith("URI:MDMF:") or to_file.startswith("URI:SSK:"):
url = nodeurl + "uri/%s" % urllib.quote(to_file)
url = nodeurl + "uri/%s" % url_quote(to_file)
else:
try:
rootcap, path = get_alias(aliases, to_file, DEFAULT_ALIAS)
except UnknownAliasError as e:
e.display(stderr)
return 1
path = unicode(path, "utf-8")
if path.startswith("/"):
suggestion = to_file.replace(u"/", u"", 1)
print("Error: The remote filename must not start with a slash", file=stderr)
print("Please try again, perhaps with %s" % quote_output(suggestion), file=stderr)
return 1
url = nodeurl + "uri/%s/" % urllib.quote(rootcap)
url = nodeurl + "uri/%s/" % url_quote(rootcap)
if path:
url += escape_path(path)
else:
@ -80,8 +84,13 @@ def put(options):
# Content-Length field. So we currently must copy it.
if verbosity > 0:
print("waiting for file data on stdin..", file=stderr)
data = stdin.read()
infileobj = StringIO(data)
# We're uploading arbitrary files, so this had better be bytes:
if PY2:
stdinb = stdin
else:
stdinb = stdin.buffer
data = stdinb.read()
infileobj = BytesIO(data)
resp = do_http("PUT", url, infileobj)

View File

@ -1,7 +1,10 @@
from past.builtins import unicode
from urllib.parse import quote as url_quote
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
UnknownAliasError
import urllib
def webopen(options, opener=None):
nodeurl = options['node-url']
@ -15,9 +18,10 @@ def webopen(options, opener=None):
except UnknownAliasError as e:
e.display(stderr)
return 1
path = unicode(path, "utf-8")
if path == '/':
path = ''
url = nodeurl + "uri/%s" % urllib.quote(rootcap)
url = nodeurl + "uri/%s" % url_quote(rootcap)
if path:
url += "/" + escape_path(path)
else:

View File

@ -1,4 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os.path, json
from twisted.trial import unittest
@ -24,12 +34,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
def test_unicode_filename(self):
self.basedir = "cli/Cp/unicode_filename"
fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
try:
fn1_arg = fn1.encode(get_io_encoding())
artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
fn1 = os.path.join(self.basedir, u"\u00C4rtonwall")
artonwall_arg = u"\u00C4rtonwall"
skip_if_cannot_represent_filename(fn1)
@ -44,15 +50,15 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
d = self.do_cli("create-alias", "tahoe")
d.addCallback(lambda res: self.do_cli("cp", fn1_arg, "tahoe:"))
d.addCallback(lambda res: self.do_cli("cp", fn1, "tahoe:"))
d.addCallback(lambda res: self.do_cli("get", "tahoe:" + artonwall_arg))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA1))
d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA1))
d.addCallback(lambda res: self.do_cli("cp", fn2, "tahoe:"))
d.addCallback(lambda res: self.do_cli("get", "tahoe:Metallica"))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
d.addCallback(lambda rc_out_err: self.assertEqual(rc_out_err[1], DATA2))
d.addCallback(lambda res: self.do_cli("ls", "tahoe:"))
def _check(args):
@ -66,8 +72,10 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessIn("files whose names could not be converted", err)
else:
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"Metallica\n\u00C4rtonwall\n")
self.failUnlessReallyEqual(err, "")
if PY2:
out = out.decode(get_io_encoding())
self.failUnlessReallyEqual(out, u"Metallica\n\u00C4rtonwall\n")
self.assertEqual(len(err), 0, err)
d.addCallback(_check)
return d
@ -98,7 +106,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
fn1 = os.path.join(self.basedir, "Metallica")
fn2 = os.path.join(outdir, "Not Metallica")
fn3 = os.path.join(outdir, "test2")
DATA1 = "puppies" * 10000
DATA1 = b"puppies" * 10000
fileutil.write(fn1, DATA1)
d = self.do_cli("create-alias", "tahoe")
@ -128,7 +136,7 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessReallyEqual(rc, 1)
self.failUnlessIn("when copying into a directory, all source files must have names, but",
err)
self.failUnlessReallyEqual(out, "")
self.assertEqual(len(out), 0, out)
d.addCallback(_resp)
# Create a directory, linked at tahoe:test .
@ -200,13 +208,8 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
def test_unicode_dirnames(self):
self.basedir = "cli/Cp/unicode_dirnames"
fn1 = os.path.join(unicode(self.basedir), u"\u00C4rtonwall")
try:
fn1_arg = fn1.encode(get_io_encoding())
del fn1_arg # hush pyflakes
artonwall_arg = u"\u00C4rtonwall".encode(get_io_encoding())
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
fn1 = os.path.join(self.basedir, u"\u00C4rtonwall")
artonwall_arg = u"\u00C4rtonwall"
skip_if_cannot_represent_filename(fn1)
@ -222,13 +225,15 @@ class Cp(GridTestMixin, CLITestMixin, unittest.TestCase):
unicode_to_output(u"\u00C4rtonwall")
except UnicodeEncodeError:
self.failUnlessReallyEqual(rc, 1)
self.failUnlessReallyEqual(out, "")
self.assertEqual(len(out), 0, out)
self.failUnlessIn(quote_output(u"\u00C4rtonwall"), err)
self.failUnlessIn("files whose names could not be converted", err)
else:
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(out.decode(get_io_encoding()), u"\u00C4rtonwall\n")
self.failUnlessReallyEqual(err, "")
if PY2:
out = out.decode(get_io_encoding())
self.failUnlessReallyEqual(out, u"\u00C4rtonwall\n")
self.assertEqual(len(err), 0, err)
d.addCallback(_check)
return d
@ -818,9 +823,9 @@ cp -r $DIRCAP5 $DIRCAP6 to : E9-COLLIDING-TARGETS
"""
class CopyOut(GridTestMixin, CLITestMixin, unittest.TestCase):
FILE_CONTENTS = "file text"
FILE_CONTENTS_5 = "5"
FILE_CONTENTS_6 = "6"
FILE_CONTENTS = b"file text"
FILE_CONTENTS_5 = b"5"
FILE_CONTENTS_6 = b"6"
def do_setup(self):
# first we build a tahoe filesystem that contains:

View File

@ -1,12 +1,26 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_str
from six.moves import StringIO
import os.path
from twisted.trial import unittest
import urllib
from urllib.parse import quote as url_quote
from allmydata.util import fileutil
from allmydata.scripts.common import get_aliases
from allmydata.scripts import cli, runner
from ..no_network import GridTestMixin
from allmydata.util.encodingutil import quote_output, get_io_encoding
from allmydata.util.encodingutil import quote_output
from .common import CLITestMixin
class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
@ -22,7 +36,7 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
rc = cli.webopen(o.subOptions, urls.append)
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(len(urls), 1)
self.failUnlessReallyEqual(urls[0], expected_url)
self.assertEqual(urls[0], expected_url)
def test_create(self):
self.basedir = "cli/CreateAlias/create"
@ -36,19 +50,19 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
self.assertIn("Alias 'tahoe' created", stdout)
aliases = get_aliases(self.get_clientdir())
self.failUnless("tahoe" in aliases)
self.failUnless(aliases["tahoe"].startswith("URI:DIR2:"))
self.failUnless(aliases["tahoe"].startswith(b"URI:DIR2:"))
d.addCallback(_done)
d.addCallback(lambda res: self.do_cli("create-alias", "two:"))
def _stash_urls(res):
aliases = get_aliases(self.get_clientdir())
node_url_file = os.path.join(self.get_clientdir(), "node.url")
nodeurl = fileutil.read(node_url_file).strip()
nodeurl = fileutil.read(node_url_file, mode="r").strip()
self.welcome_url = nodeurl
uribase = nodeurl + "uri/"
self.tahoe_url = uribase + urllib.quote(aliases["tahoe"])
self.tahoe_url = uribase + url_quote(aliases["tahoe"])
self.tahoe_subdir_url = self.tahoe_url + "/subdir"
self.two_url = uribase + urllib.quote(aliases["two"])
self.two_url = uribase + url_quote(aliases["two"])
self.two_uri = aliases["two"]
d.addCallback(_stash_urls)
@ -128,13 +142,13 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
# like a valid dircap, so get_aliases() will raise an exception.
aliases = get_aliases(self.get_clientdir())
self.failUnless("added" in aliases)
self.failUnless(aliases["added"].startswith("URI:DIR2:"))
self.failUnless(aliases["added"].startswith(b"URI:DIR2:"))
# to be safe, let's confirm that we don't see "NAME2:" in CAP1.
# No chance of a false-negative, because the hyphen in
# "un-corrupted1" is not a valid base32 character.
self.failIfIn("un-corrupted1:", aliases["added"])
self.failIfIn(b"un-corrupted1:", aliases["added"])
self.failUnless("un-corrupted1" in aliases)
self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:"))
self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:"))
d.addCallback(_check_not_corrupted1)
def _remove_trailing_newline_and_add_alias(ign):
@ -149,10 +163,10 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failIf(stderr)
aliases = get_aliases(self.get_clientdir())
self.failUnless("un-corrupted1" in aliases)
self.failUnless(aliases["un-corrupted1"].startswith("URI:DIR2:"))
self.failIfIn("un-corrupted2:", aliases["un-corrupted1"])
self.failUnless(aliases["un-corrupted1"].startswith(b"URI:DIR2:"))
self.failIfIn(b"un-corrupted2:", aliases["un-corrupted1"])
self.failUnless("un-corrupted2" in aliases)
self.failUnless(aliases["un-corrupted2"].startswith("URI:DIR2:"))
self.failUnless(aliases["un-corrupted2"].startswith(b"URI:DIR2:"))
d.addCallback(_check_not_corrupted)
return d
@ -160,61 +174,62 @@ class CreateAlias(GridTestMixin, CLITestMixin, unittest.TestCase):
self.basedir = "cli/CreateAlias/create_unicode"
self.set_up_grid(oneshare=True)
try:
etudes_arg = u"\u00E9tudes".encode(get_io_encoding())
lumiere_arg = u"lumi\u00E8re.txt".encode(get_io_encoding())
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
etudes_arg = u"\u00E9tudes"
lumiere_arg = u"lumi\u00E8re.txt"
d = self.do_cli("create-alias", etudes_arg)
def _check_create_unicode(args):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(err, "")
self.failUnlessIn("Alias %s created" % quote_output(u"\u00E9tudes"), out)
self.assertEqual(len(err), 0, err)
self.failUnlessIn(ensure_str("Alias %s created") % quote_output(etudes_arg), out)
aliases = get_aliases(self.get_clientdir())
self.failUnless(aliases[u"\u00E9tudes"].startswith("URI:DIR2:"))
self.failUnless(aliases[u"\u00E9tudes"].startswith(b"URI:DIR2:"))
d.addCallback(_check_create_unicode)
d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":"))
def _check_ls1(args):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(out, "")
self.assertEqual(len(err), 0, err)
self.assertEqual(len(out), 0, out)
d.addCallback(_check_ls1)
DATA = b"Blah blah blah \xff blah \x00 blah"
d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":uploaded.txt",
stdin="Blah blah blah"))
stdin=DATA))
d.addCallback(lambda res: self.do_cli("ls", etudes_arg + ":"))
def _check_ls2(args):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(out, "uploaded.txt\n")
self.assertEqual(len(err), 0, err)
self.assertEqual(out, "uploaded.txt\n")
d.addCallback(_check_ls2)
d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt"))
d.addCallback(lambda res: self.do_cli("get", etudes_arg + ":uploaded.txt",
return_bytes=True))
def _check_get(args):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(out, "Blah blah blah")
self.assertEqual(len(err), 0, err)
self.failUnlessReallyEqual(out, DATA)
d.addCallback(_check_get)
# Ensure that an Unicode filename in an Unicode alias works as expected
d.addCallback(lambda res: self.do_cli("put", "-", etudes_arg + ":" + lumiere_arg,
stdin="Let the sunshine In!"))
stdin=b"Let the sunshine In!"))
d.addCallback(lambda res: self.do_cli("get",
get_aliases(self.get_clientdir())[u"\u00E9tudes"] + "/" + lumiere_arg))
d.addCallback(lambda res: self.do_cli(
"get",
str(get_aliases(self.get_clientdir())[u"\u00E9tudes"], "ascii") + "/" + lumiere_arg,
return_bytes=True))
def _check_get2(args):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 0)
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(out, "Let the sunshine In!")
self.assertEqual(len(err), 0, err)
self.failUnlessReallyEqual(out, b"Let the sunshine In!")
d.addCallback(_check_get2)
return d

View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os.path
from twisted.trial import unittest
from twisted.python import usage
@ -17,7 +29,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
# tahoe get `echo DATA | tahoe put`
# tahoe get `echo DATA | tahoe put -`
self.basedir = "cli/Put/unlinked_immutable_stdin"
DATA = "data" * 100
DATA = b"data\xff" * 100
self.set_up_grid(oneshare=True)
d = self.do_cli("put", stdin=DATA)
def _uploaded(res):
@ -27,10 +39,11 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.readcap = out
self.failUnless(self.readcap.startswith("URI:CHK:"))
d.addCallback(_uploaded)
d.addCallback(lambda res: self.do_cli("get", self.readcap))
d.addCallback(lambda res: self.do_cli("get", self.readcap,
return_bytes=True))
def _downloaded(res):
(rc, out, err) = res
self.failUnlessReallyEqual(err, "")
self.failUnlessReallyEqual(err, b"")
self.failUnlessReallyEqual(out, DATA)
d.addCallback(_downloaded)
d.addCallback(lambda res: self.do_cli("put", "-", stdin=DATA))
@ -46,10 +59,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.basedir = "cli/Put/unlinked_immutable_from_file"
self.set_up_grid(oneshare=True)
rel_fn = unicode(os.path.join(self.basedir, "DATAFILE"))
rel_fn = str(os.path.join(self.basedir, "DATAFILE"))
abs_fn = abspath_expanduser_unicode(rel_fn)
# we make the file small enough to fit in a LIT file, for speed
fileutil.write(rel_fn, "short file")
fileutil.write(rel_fn, b"short file has some bytes \xff yes")
d = self.do_cli_unicode(u"put", [rel_fn])
def _uploaded(args):
(rc, out, err) = args
@ -79,8 +92,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
rel_fn = os.path.join(self.basedir, "DATAFILE")
# we make the file small enough to fit in a LIT file, for speed
DATA = "short file"
DATA2 = "short file two"
DATA = b"short file"
DATA2 = b"short file two"
fileutil.write(rel_fn, DATA)
d = self.do_cli("create-alias", "tahoe")
@ -95,7 +108,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.readcap = readcap
d.addCallback(_uploaded)
d.addCallback(lambda res:
self.do_cli("get", "tahoe:uploaded.txt"))
self.do_cli("get", "tahoe:uploaded.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
@ -110,32 +124,36 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
d.addCallback(lambda res:
self.do_cli("put", rel_fn, "subdir/uploaded2.txt"))
d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt"))
d.addCallback(lambda res: self.do_cli("get", "subdir/uploaded2.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
d.addCallback(lambda res:
self.do_cli("put", rel_fn, "tahoe:uploaded3.txt"))
d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt"))
d.addCallback(lambda res: self.do_cli("get", "tahoe:uploaded3.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
d.addCallback(lambda res:
self.do_cli("put", rel_fn, "tahoe:subdir/uploaded4.txt"))
d.addCallback(lambda res:
self.do_cli("get", "tahoe:subdir/uploaded4.txt"))
self.do_cli("get", "tahoe:subdir/uploaded4.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
def _get_dircap(res):
self.dircap = get_aliases(self.get_clientdir())["tahoe"]
self.dircap = str(get_aliases(self.get_clientdir())["tahoe"], "ascii")
d.addCallback(_get_dircap)
d.addCallback(lambda res:
self.do_cli("put", rel_fn,
self.dircap+":./uploaded5.txt"))
d.addCallback(lambda res:
self.do_cli("get", "tahoe:uploaded5.txt"))
self.do_cli("get", "tahoe:uploaded5.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
@ -143,7 +161,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.do_cli("put", rel_fn,
self.dircap+":./subdir/uploaded6.txt"))
d.addCallback(lambda res:
self.do_cli("get", "tahoe:subdir/uploaded6.txt"))
self.do_cli("get", "tahoe:subdir/uploaded6.txt",
return_bytes=True))
d.addCallback(lambda rc_stdout_stderr:
self.failUnlessReallyEqual(rc_stdout_stderr[1], DATA))
@ -158,10 +177,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.basedir = "cli/Put/mutable_unlinked"
self.set_up_grid(oneshare=True)
DATA = "data" * 100
DATA2 = "two" * 100
DATA = b"data" * 100
DATA2 = b"two" * 100
rel_fn = os.path.join(self.basedir, "DATAFILE")
DATA3 = "three" * 100
DATA3 = b"three" * 100
fileutil.write(rel_fn, DATA3)
d = self.do_cli("put", "--mutable", stdin=DATA)
@ -172,7 +191,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.filecap = out
self.failUnless(self.filecap.startswith("URI:SSK:"), self.filecap)
d.addCallback(_created)
d.addCallback(lambda res: self.do_cli("get", self.filecap))
d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA))
d.addCallback(lambda res: self.do_cli("put", "-", self.filecap, stdin=DATA2))
@ -182,7 +201,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessIn("200 OK", err)
self.failUnlessReallyEqual(self.filecap, out)
d.addCallback(_replaced)
d.addCallback(lambda res: self.do_cli("get", self.filecap))
d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
d.addCallback(lambda res: self.do_cli("put", rel_fn, self.filecap))
@ -191,7 +210,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessIn("200 OK", err)
self.failUnlessReallyEqual(self.filecap, out)
d.addCallback(_replaced2)
d.addCallback(lambda res: self.do_cli("get", self.filecap))
d.addCallback(lambda res: self.do_cli("get", self.filecap, return_bytes=True))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA3))
return d
@ -204,10 +223,10 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.basedir = "cli/Put/mutable"
self.set_up_grid(oneshare=True)
DATA1 = "data" * 100
DATA1 = b"data" * 100
fn1 = os.path.join(self.basedir, "DATA1")
fileutil.write(fn1, DATA1)
DATA2 = "two" * 100
DATA2 = b"two\xff" * 100
fn2 = os.path.join(self.basedir, "DATA2")
fileutil.write(fn2, DATA2)
@ -229,7 +248,7 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
self.failUnlessEqual(out, self.uri, str(res))
d.addCallback(_check2)
d.addCallback(lambda res:
self.do_cli("get", "tahoe:uploaded.txt"))
self.do_cli("get", "tahoe:uploaded.txt", return_bytes=True))
d.addCallback(lambda rc_out_err: self.failUnlessReallyEqual(rc_out_err[1], DATA2))
return d
@ -429,26 +448,23 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
(rc, out, err) = args
self.failUnlessReallyEqual(rc, 1)
self.failUnlessIn("error:", err)
self.failUnlessReallyEqual(out, "")
self.assertEqual(len(out), 0, out)
d.addCallback(_check)
return d
def test_immutable_from_file_unicode(self):
# tahoe put "\u00E0 trier.txt" "\u00E0 trier.txt"
try:
a_trier_arg = u"\u00E0 trier.txt".encode(get_io_encoding())
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII command argument could not be encoded on this platform.")
a_trier_arg = u"\u00E0 trier.txt"
skip_if_cannot_represent_filename(u"\u00E0 trier.txt")
self.basedir = "cli/Put/immutable_from_file_unicode"
self.set_up_grid(oneshare=True)
rel_fn = os.path.join(unicode(self.basedir), u"\u00E0 trier.txt")
rel_fn = os.path.join(str(self.basedir), u"\u00E0 trier.txt")
# we make the file small enough to fit in a LIT file, for speed
DATA = "short file"
DATA = b"short file \xff bytes"
fileutil.write(rel_fn, DATA)
d = self.do_cli("create-alias", "tahoe")
@ -464,7 +480,8 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
d.addCallback(_uploaded)
d.addCallback(lambda res:
self.do_cli("get", "tahoe:" + a_trier_arg))
self.do_cli("get", "tahoe:" + a_trier_arg,
return_bytes=True))
d.addCallback(lambda rc_out_err:
self.failUnlessReallyEqual(rc_out_err[1], DATA))

View File

@ -6,7 +6,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2, bchr, binary_type
from future.utils import PY2, PY3, bchr, binary_type
from future.builtins import str as future_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401
@ -15,7 +15,8 @@ import os
import time
import signal
from random import randrange
from six.moves import StringIO
if PY2:
from StringIO import StringIO
from io import (
TextIOWrapper,
BytesIO,
@ -64,23 +65,28 @@ def run_cli_native(verb, *args, **kwargs):
Most code should prefer ``run_cli_unicode`` which deals with all the
necessary encoding considerations.
:param native_str verb: The command to run. For example, ``"create-node"``.
:param native_str verb: The command to run. For example,
``"create-node"``.
:param [native_str] args: The arguments to pass to the command. For example,
``("--hostname=localhost",)``.
:param [native_str] args: The arguments to pass to the command. For
example, ``("--hostname=localhost",)``.
:param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable
before ``verb``.
:param [native_str] nodeargs: Extra arguments to pass to the Tahoe
executable before ``verb``.
:param native_str stdin: Text to pass to the command via stdin.
:param bytes|unicode stdin: Text or bytes to pass to the command via stdin.
:param NoneType|str encoding: The name of an encoding which stdout and
stderr will be configured to use. ``None`` means stdout and stderr
will accept bytes and unicode and use the default system encoding for
translating between them.
stderr will be configured to use. ``None`` means matching default
behavior for the given Python version.
:param bool return_bytes: If False, stdout/stderr is native string,
matching native behavior. If True, stdout/stderr are returned as
bytes.
"""
nodeargs = kwargs.pop("nodeargs", [])
encoding = kwargs.pop("encoding", None)
encoding = kwargs.pop("encoding", "utf-8")
return_bytes = kwargs.pop("return_bytes", False)
verb = maybe_unicode_to_argv(verb)
args = [maybe_unicode_to_argv(a) for a in args]
nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs]
@ -93,36 +99,42 @@ def run_cli_native(verb, *args, **kwargs):
)
argv = nodeargs + [verb] + list(args)
stdin = kwargs.get("stdin", "")
if encoding is None:
if PY2:
# The original behavior, the Python 2 behavior, is to accept either
# bytes or unicode and try to automatically encode or decode as
# necessary. This works okay for ASCII and if LANG is set
# appropriately. These aren't great constraints so we should move
# away from this behavior.
stdout = StringIO()
stderr = StringIO()
else:
# Default on Python 3 is accepting text.
stdout = TextIOWrapper(BytesIO(), "utf-8")
stderr = TextIOWrapper(BytesIO(), "utf-8")
if PY2:
# The original behavior, the Python 2 behavior, is to accept either
# bytes or unicode and try to automatically encode or decode as
# necessary. This works okay for ASCII and if LANG is set
# appropriately. These aren't great constraints so we should move
# away from this behavior.
stdin = StringIO(stdin)
stdout = StringIO()
stderr = StringIO()
else:
# The new behavior, the Python 3 behavior, is to accept unicode and
# encode it using a specific encoding. For older versions of Python
# 3, the encoding is determined from LANG (bad) but for newer Python
# 3, the encoding is always utf-8 (good). Tests can pass in different
# encodings to exercise different behaviors.
# encode it using a specific encoding. For older versions of Python 3,
# the encoding is determined from LANG (bad) but for newer Python 3,
# the encoding is either LANG if it supports full Unicode, otherwise
# utf-8 (good). Tests can pass in different encodings to exercise
# different behaviors.
if isinstance(stdin, str):
stdin = stdin.encode(encoding)
stdin = TextIOWrapper(BytesIO(stdin), encoding)
stdout = TextIOWrapper(BytesIO(), encoding)
stderr = TextIOWrapper(BytesIO(), encoding)
d = defer.succeed(argv)
d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout)
d.addCallback(runner.dispatch,
stdin=StringIO(stdin),
stdin=stdin,
stdout=stdout, stderr=stderr)
def _done(rc):
def _done(rc, stdout=stdout, stderr=stderr):
if return_bytes and PY3:
stdout = stdout.buffer
stderr = stderr.buffer
return 0, _getvalue(stdout), _getvalue(stderr)
def _err(f):
def _err(f, stdout=stdout, stderr=stderr):
f.trap(SystemExit)
if return_bytes and PY3:
stdout = stdout.buffer
stderr = stderr.buffer
return f.value.code, _getvalue(stdout), _getvalue(stderr)
d.addCallbacks(_done, _err)
return d

View File

@ -177,8 +177,11 @@ PORTED_TEST_MODULES = [
"allmydata.test.cli.test_backup",
"allmydata.test.cli.test_backupdb",
"allmydata.test.cli.test_check",
"allmydata.test.cli.test_cp",
"allmydata.test.cli.test_create",
"allmydata.test.cli.test_create_alias",
"allmydata.test.cli.test_invite",
"allmydata.test.cli.test_put",
"allmydata.test.cli.test_status",
"allmydata.test.mutable.test_checker",