ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-29 17:28:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3552.test_system-python-3' into 3565.web-tests-python-3-part-1

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2020-12-17 09:51:52 -05:00
commit 9b59e7e245
16 changed files with 200 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3533.minor Normal file
View File

0
newsfragments/3552.minor Normal file
View File

5
nix/tahoe-lafs.nix
View File

@ -23,17 +23,12 @@ python.pkgs.buildPythonPackage rec {
# This list is over-zealous because it's more work to disable individual
# tests with in a module.
# test_system is a lot of integration-style tests that do a lot of real
# networking between many processes. They sometimes fail spuriously.
rm src/allmydata/test/test_system.py
# Many of these tests don't properly skip when i2p or tor dependencies are
# not supplied (and we are not supplying them).
rm src/allmydata/test/test_i2p_provider.py
rm src/allmydata/test/test_connections.py
rm src/allmydata/test/cli/test_create.py
rm src/allmydata/test/test_client.py
rm src/allmydata/test/test_runner.py
'';

6
src/allmydata/client.py
View File

@ -714,7 +714,7 @@ class _Client(node.Node, pollmixin.PollMixin):
def get_long_nodeid(self):
# this matches what IServer.get_longname() says about us elsewhere
vk_string = ed25519.string_from_verifying_key(self._node_public_key)
return remove_prefix(vk_string, "pub-")
return remove_prefix(vk_string, b"pub-")
def get_long_tubid(self):
return idlib.nodeid_b2a(self.nodeid)
@ -898,10 +898,6 @@ class _Client(node.Node, pollmixin.PollMixin):
if helper_furl in ("None", ""):
helper_furl = None
# FURLs need to be bytes:
if helper_furl is not None:
helper_furl = helper_furl.encode("utf-8")
DEP = self.encoding_params
DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))

3
src/allmydata/immutable/downloader/finder.py
View File

@ -9,6 +9,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_str
import time
now = time.time
@ -98,7 +99,7 @@ class ShareFinder(object):
# internal methods
def loop(self):
pending_s = ",".join([rt.server.get_name()
pending_s = ",".join([ensure_str(rt.server.get_name())
for rt in self.pending_requests]) # sort?
self.log(format="ShareFinder loop: running=%(running)s"
" hungry=%(hungry)s, pending=%(pending)s",

3
src/allmydata/immutable/upload.py
View File

@ -11,6 +11,7 @@ from future.utils import PY2, native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long, unicode
from six import ensure_str
import os, time, weakref, itertools
from zope.interface import implementer
@ -1825,7 +1826,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
def startService(self):
service.MultiService.startService(self)
if self._helper_furl:
self.parent.tub.connectTo(self._helper_furl,
self.parent.tub.connectTo(ensure_str(self._helper_furl),
self._got_helper)
def _got_helper(self, helper):

6
src/allmydata/introducer/client.py
View File

@ -11,7 +11,7 @@ if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long
from six import ensure_text
from six import ensure_text, ensure_str
import time
from zope.interface import implementer
@ -39,8 +39,6 @@ class IntroducerClient(service.Service, Referenceable):
nickname, my_version, oldest_supported,
sequencer, cache_filepath):
self._tub = tub
if isinstance(introducer_furl, str):
introducer_furl = introducer_furl.encode("utf-8")
self.introducer_furl = introducer_furl
assert isinstance(nickname, str)
@ -96,7 +94,7 @@ class IntroducerClient(service.Service, Referenceable):
def startService(self):
service.Service.startService(self)
self._introducer_error = None
rc = self._tub.connectTo(self.introducer_furl, self._got_introducer)
rc = self._tub.connectTo(ensure_str(self.introducer_furl), self._got_introducer)
self._introducer_reconnector = rc
def connect_failed(failure):
self.log("Initial Introducer connection failed: perhaps it's down",

34
src/allmydata/test/common_util.py
View File

@ -1,5 +1,8 @@
from __future__ import print_function
from future.utils import PY2, native_str
from future.builtins import str as future_str
import os
import time
import signal
@ -48,24 +51,23 @@ def _getvalue(io):
return io.read()
def run_cli_bytes(verb, *args, **kwargs):
def run_cli_native(verb, *args, **kwargs):
"""
Run a Tahoe-LAFS CLI command specified as bytes.
Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode
(on Python 3); basically, it accepts a native string.
Most code should prefer ``run_cli_unicode`` which deals with all the
necessary encoding considerations. This helper still exists so that novel
misconfigurations can be explicitly tested (for example, receiving UTF-8
bytes when the system encoding claims to be ASCII).
necessary encoding considerations.
:param bytes verb: The command to run. For example, ``b"create-node"``.
:param native_str verb: The command to run. For example, ``b"create-node"``.
:param [bytes] args: The arguments to pass to the command. For example,
:param [native_str] args: The arguments to pass to the command. For example,
``(b"--hostname=localhost",)``.
:param [bytes] nodeargs: Extra arguments to pass to the Tahoe executable
:param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable
before ``verb``.
:param bytes stdin: Text to pass to the command via stdin.
:param native_str stdin: Text to pass to the command via stdin.
:param NoneType|str encoding: The name of an encoding which stdout and
stderr will be configured to use. ``None`` means stdout and stderr
@ -75,8 +77,8 @@ def run_cli_bytes(verb, *args, **kwargs):
nodeargs = kwargs.pop("nodeargs", [])
encoding = kwargs.pop("encoding", None)
precondition(
all(isinstance(arg, bytes) for arg in [verb] + nodeargs + list(args)),
"arguments to run_cli must be bytes -- convert using unicode_to_argv",
all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)),
"arguments to run_cli must be a native string -- convert using unicode_to_argv",
verb=verb,
args=args,
nodeargs=nodeargs,
@ -135,15 +137,19 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
if nodeargs is None:
nodeargs = []
precondition(
all(isinstance(arg, unicode) for arg in [verb] + nodeargs + argv),
all(isinstance(arg, future_str) for arg in [verb] + nodeargs + argv),
"arguments to run_cli_unicode must be unicode",
verb=verb,
nodeargs=nodeargs,
argv=argv,
)
codec = encoding or "ascii"
if PY2:
encode = lambda t: None if t is None else t.encode(codec)
d = run_cli_bytes(
else:
# On Python 3 command-line parsing expects Unicode!
encode = lambda t: t
d = run_cli_native(
encode(verb),
nodeargs=list(encode(arg) for arg in nodeargs),
stdin=encode(stdin),
@ -161,7 +167,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
return d
run_cli = run_cli_bytes
run_cli = run_cli_native
def parse_cli(*argv):

7
src/allmydata/test/common_web.py
View File

@ -47,12 +47,17 @@ class VerboseError(Error):
@inlineCallbacks
def do_http(method, url, **kwargs):
"""
Run HTTP query, return Deferred of body as bytes.
"""
response = yield treq.request(method, url, persistent=False, **kwargs)
body = yield treq.content(response)
# TODO: replace this with response.fail_for_status when
# https://github.com/twisted/treq/pull/159 has landed
if 400 <= response.code < 600:
raise VerboseError(response.code, response=body)
raise VerboseError(
response.code, response="For request {} to {}, got: {}".format(
method, url, body))
returnValue(body)

168
src/allmydata/test/test_system.py
View File

@ -1,7 +1,22 @@
"""
Ported to Python 3, partially: test_filesystem* will be done in a future round.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2, PY3
if PY2:
# Don't import bytes since it causes issues on (so far unported) modules on Python 2.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401
from past.builtins import chr as byteschr, long
from six import ensure_text, ensure_str
import os, re, sys, time, json
from functools import partial
from unittest import skipIf
from bs4 import BeautifulSoup
@ -40,7 +55,7 @@ from .common import (
TEST_RSA_KEY_SIZE,
SameProcessStreamEndpointAssigner,
)
from .common_web import do_http, Error
from .common_web import do_http as do_http_bytes, Error
from .web.common import (
assert_soup_has_tag_with_attributes
)
@ -48,12 +63,28 @@ from .web.common import (
# TODO: move this to common or common_util
from allmydata.test.test_runner import RunBinTahoeMixin
from . import common_util as testutil
from .common_util import run_cli
from .common_util import run_cli_unicode
from ..scripts.common import (
write_introducer,
)
LARGE_DATA = """
def run_cli(*args, **kwargs):
"""
Backwards compatible version so we don't have to change all the tests.
"""
nodeargs = [ensure_text(a) for a in kwargs.pop("nodeargs", [])]
kwargs["nodeargs"] = nodeargs
return run_cli_unicode(
ensure_text(args[0]), [ensure_text(a) for a in args[1:]], **kwargs)
def do_http(*args, **kwargs):
"""Wrapper for do_http() that returns Unicode."""
return do_http_bytes(*args, **kwargs).addCallback(
lambda b: str(b, "utf-8"))
LARGE_DATA = b"""
This is some data to publish to the remote grid.., which needs to be large
enough to not fit inside a LIT uri.
"""
@ -629,7 +660,7 @@ def _render_config(config):
"""
Convert a ``dict`` of ``dict`` of ``bytes`` to an ini-format string.
"""
return "\n\n".join(list(
return u"\n\n".join(list(
_render_config_section(k, v)
for (k, v)
in config.items()
@ -640,7 +671,7 @@ def _render_config_section(heading, values):
Convert a ``bytes`` heading and a ``dict`` of ``bytes`` to an ini-format
section as ``bytes``.
"""
return "[{}]\n{}\n".format(
return u"[{}]\n{}\n".format(
heading, _render_section_values(values)
)
@ -649,8 +680,8 @@ def _render_section_values(values):
Convert a ``dict`` of ``bytes`` to the body of an ini-format section as
``bytes``.
"""
return "\n".join(list(
"{} = {}".format(k, v)
return u"\n".join(list(
u"{} = {}".format(k, v)
for (k, v)
in sorted(values.items())
))
@ -753,7 +784,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
self.helper_furl = helper_furl
if self.numclients >= 4:
with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+') as f:
with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f:
f.write(
"[client]\n"
"helper.furl = {}\n".format(helper_furl)
@ -796,8 +827,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
def setconf(config, which, section, feature, value):
if which in feature_matrix.get((section, feature), {which}):
if isinstance(value, unicode):
value = value.encode("utf-8")
config.setdefault(section, {})[feature] = value
setnode = partial(setconf, config, which, "node")
@ -870,7 +899,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
config = "[client]\n"
if helper_furl:
config += "helper.furl = %s\n" % helper_furl
basedir.child("tahoe.cfg").setContent(config)
basedir.child("tahoe.cfg").setContent(config.encode("utf-8"))
private = basedir.child("private")
private.makedirs()
write_introducer(
@ -980,12 +1009,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def test_upload_and_download_convergent(self):
self.basedir = "system/SystemTest/test_upload_and_download_convergent"
return self._test_upload_and_download(convergence="some convergence string")
return self._test_upload_and_download(convergence=b"some convergence string")
def _test_upload_and_download(self, convergence):
# we use 4000 bytes of data, which will result in about 400k written
# to disk among all our simulated nodes
DATA = "Some data to upload\n" * 200
DATA = b"Some data to upload\n" * 200
d = self.set_up_nodes()
def _check_connections(res):
for c in self.clients:
@ -993,7 +1022,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
all_peerids = c.get_storage_broker().get_all_serverids()
self.failUnlessEqual(len(all_peerids), self.numclients)
sb = c.storage_broker
permuted_peers = sb.get_servers_for_psi("a")
permuted_peers = sb.get_servers_for_psi(b"a")
self.failUnlessEqual(len(permuted_peers), self.numclients)
d.addCallback(_check_connections)
@ -1016,7 +1045,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
theuri = results.get_uri()
log.msg("upload finished: uri is %s" % (theuri,))
self.uri = theuri
assert isinstance(self.uri, str), self.uri
assert isinstance(self.uri, bytes), self.uri
self.cap = uri.from_string(self.uri)
self.n = self.clients[1].create_node_from_uri(self.uri)
d.addCallback(_upload_done)
@ -1050,17 +1079,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(lambda ign:
n.read(MemoryConsumer(), offset=1, size=4))
def _read_portion_done(mc):
self.failUnlessEqual("".join(mc.chunks), DATA[1:1+4])
self.failUnlessEqual(b"".join(mc.chunks), DATA[1:1+4])
d.addCallback(_read_portion_done)
d.addCallback(lambda ign:
n.read(MemoryConsumer(), offset=2, size=None))
def _read_tail_done(mc):
self.failUnlessEqual("".join(mc.chunks), DATA[2:])
self.failUnlessEqual(b"".join(mc.chunks), DATA[2:])
d.addCallback(_read_tail_done)
d.addCallback(lambda ign:
n.read(MemoryConsumer(), size=len(DATA)+1000))
def _read_too_much(mc):
self.failUnlessEqual("".join(mc.chunks), DATA)
self.failUnlessEqual(b"".join(mc.chunks), DATA)
d.addCallback(_read_too_much)
return d
@ -1110,7 +1139,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return connected
d.addCallback(lambda ign: self.poll(_has_helper))
HELPER_DATA = "Data that needs help to upload" * 1000
HELPER_DATA = b"Data that needs help to upload" * 1000
def _upload_with_helper(res):
u = upload.Data(HELPER_DATA, convergence=convergence)
d = self.extra_node.upload(u)
@ -1144,7 +1173,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(fireEventually)
def _upload_resumable(res):
DATA = "Data that needs help to upload and gets interrupted" * 1000
DATA = b"Data that needs help to upload and gets interrupted" * 1000
u1 = CountingDataUploadable(DATA, convergence=convergence)
u2 = CountingDataUploadable(DATA, convergence=convergence)
@ -1266,7 +1295,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
s = stats["stats"]
self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1)
c = stats["counters"]
self.failUnless("storage_server.allocate" in c)
# Probably this should be Unicode eventually? But we haven't ported
# stats code yet.
self.failUnless(b"storage_server.allocate" in c)
d.addCallback(_grab_stats)
return d
@ -1287,7 +1318,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
assert pieces[-5].startswith("client")
client_num = int(pieces[-5][-1])
storage_index_s = pieces[-1]
storage_index = si_a2b(storage_index_s)
storage_index = si_a2b(storage_index_s.encode("ascii"))
for sharename in filenames:
shnum = int(sharename)
filename = os.path.join(dirpath, sharename)
@ -1320,7 +1351,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
elif which == "signature":
signature = self.flip_bit(signature)
elif which == "share_hash_chain":
nodenum = share_hash_chain.keys()[0]
nodenum = list(share_hash_chain.keys())[0]
share_hash_chain[nodenum] = self.flip_bit(share_hash_chain[nodenum])
elif which == "block_hash_tree":
block_hash_tree[-1] = self.flip_bit(block_hash_tree[-1])
@ -1343,11 +1374,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def test_mutable(self):
self.basedir = "system/SystemTest/test_mutable"
DATA = "initial contents go here." # 25 bytes % 3 != 0
DATA = b"initial contents go here." # 25 bytes % 3 != 0
DATA_uploadable = MutableData(DATA)
NEWDATA = "new contents yay"
NEWDATA = b"new contents yay"
NEWDATA_uploadable = MutableData(NEWDATA)
NEWERDATA = "this is getting old"
NEWERDATA = b"this is getting old"
NEWERDATA_uploadable = MutableData(NEWERDATA)
d = self.set_up_nodes()
@ -1396,7 +1427,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
self.failUnless(" share_hash_chain: " in output)
self.failUnless(" block_hash_tree: 1 nodes\n" in output)
expected = (" verify-cap: URI:SSK-Verifier:%s:" %
base32.b2a(storage_index))
str(base32.b2a(storage_index), "ascii"))
self.failUnless(expected in output)
except unittest.FailTest:
print()
@ -1475,7 +1506,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
for (client_num, storage_index, filename, shnum)
in shares ])
assert len(where) == 10 # this test is designed for 3-of-10
for shnum, filename in where.items():
for shnum, filename in list(where.items()):
# shares 7,8,9 are left alone. read will check
# (share_hash_chain, block_hash_tree, share_data). New
# seqnum+R pairs will trigger a check of (seqnum, R, IV,
@ -1525,9 +1556,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
def _check_empty_file(res):
# make sure we can create empty files, this usually screws up the
# segsize math
d1 = self.clients[2].create_mutable_file(MutableData(""))
d1 = self.clients[2].create_mutable_file(MutableData(b""))
d1.addCallback(lambda newnode: newnode.download_best_version())
d1.addCallback(lambda res: self.failUnlessEqual("", res))
d1.addCallback(lambda res: self.failUnlessEqual(b"", res))
return d1
d.addCallback(_check_empty_file)
@ -1550,7 +1581,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
def flip_bit(self, good):
return good[:-1] + chr(ord(good[-1]) ^ 0x01)
return good[:-1] + byteschr(ord(good[-1:]) ^ 0x01)
def mangle_uri(self, gooduri):
# change the key, which changes the storage index, which means we'll
@ -1571,6 +1602,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# the key, which should cause the download to fail the post-download
# plaintext_hash check.
@skipIf(PY3, "Python 3 web support hasn't happened yet.")
def test_filesystem(self):
self.basedir = "system/SystemTest/test_filesystem"
self.data = LARGE_DATA
@ -1632,7 +1664,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d1.addCallback(self.log, "publish finished")
def _stash_uri(filenode):
self.uri = filenode.get_uri()
assert isinstance(self.uri, str), (self.uri, filenode)
assert isinstance(self.uri, bytes), (self.uri, filenode)
d1.addCallback(_stash_uri)
return d1
d.addCallback(_made_subdir1)
@ -1650,7 +1682,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return res
def _do_publish_private(self, res):
self.smalldata = "sssh, very secret stuff"
self.smalldata = b"sssh, very secret stuff"
ut = upload.Data(self.smalldata, convergence=None)
d = self.clients[0].create_dirnode()
d.addCallback(self.log, "GOT private directory")
@ -1737,7 +1769,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope"))
d1.addCallback(self.log, "doing add_file(ro)")
ut = upload.Data("I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence="99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
ut = upload.Data(b"I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence=b"99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut))
d1.addCallback(self.log, "doing get(ro)")
@ -1801,7 +1833,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
"largest-directory-children": 3,
"largest-immutable-file": 112,
}
for k,v in expected.iteritems():
for k,v in list(expected.items()):
self.failUnlessEqual(stats[k], v,
"stats[%s] was %s, not %s" %
(k, stats[k], v))
@ -1850,33 +1882,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return do_http("get", self.webish_url + urlpath)
def POST(self, urlpath, use_helper=False, **fields):
sepbase = "boogabooga"
sep = "--" + sepbase
sepbase = b"boogabooga"
sep = b"--" + sepbase
form = []
form.append(sep)
form.append('Content-Disposition: form-data; name="_charset"')
form.append('')
form.append('UTF-8')
form.append(b'Content-Disposition: form-data; name="_charset"')
form.append(b'')
form.append(b'UTF-8')
form.append(sep)
for name, value in fields.iteritems():
for name, value in fields.items():
if isinstance(value, tuple):
filename, value = value
form.append('Content-Disposition: form-data; name="%s"; '
'filename="%s"' % (name, filename.encode("utf-8")))
form.append(b'Content-Disposition: form-data; name="%s"; '
b'filename="%s"' % (name, filename.encode("utf-8")))
else:
form.append('Content-Disposition: form-data; name="%s"' % name)
form.append('')
form.append(str(value))
form.append(b'Content-Disposition: form-data; name="%s"' % name)
form.append(b'')
form.append(b"%s" % (value,))
form.append(sep)
form[-1] += "--"
body = ""
form[-1] += b"--"
body = b""
headers = {}
if fields:
body = "\r\n".join(form) + "\r\n"
headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
body = b"\r\n".join(form) + b"\r\n"
headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii")
return self.POST2(urlpath, body, headers, use_helper)
def POST2(self, urlpath, body="", headers={}, use_helper=False):
def POST2(self, urlpath, body=b"", headers={}, use_helper=False):
if use_helper:
url = self.helper_webish_url + urlpath
else:
@ -1884,7 +1916,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return do_http("post", url, data=body, headers=headers)
def _test_web(self, res):
public = "uri/" + self._root_directory_uri
public = "uri/" + str(self._root_directory_uri, "ascii")
d = self.GET("")
def _got_welcome(page):
html = page.replace('\n', ' ')
@ -1893,7 +1925,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
"I didn't see the right '%s' message in:\n%s" % (connected_re, page))
# nodeids/tubids don't have any regexp-special characters
nodeid_re = r'<th>Node ID:</th>\s*<td title="TubID: %s">%s</td>' % (
self.clients[0].get_long_tubid(), self.clients[0].get_long_nodeid())
self.clients[0].get_long_tubid(), str(self.clients[0].get_long_nodeid(), "ascii"))
self.failUnless(re.search(nodeid_re, html),
"I didn't see the right '%s' message in:\n%s" % (nodeid_re, page))
self.failUnless("Helper: 0 active uploads" in page)
@ -1954,7 +1986,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# upload a file with PUT
d.addCallback(self.log, "about to try PUT")
d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
"new.txt contents"))
b"new.txt contents"))
d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
d.addCallback(self.failUnlessEqual, "new.txt contents")
# and again with something large enough to use multiple segments,
@ -1965,23 +1997,23 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
c.encoding_params['happy'] = 1
d.addCallback(_new_happy_semantics)
d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
"big" * 500000)) # 1.5MB
b"big" * 500000)) # 1.5MB
d.addCallback(lambda res: self.GET(public + "/subdir3/big.txt"))
d.addCallback(lambda res: self.failUnlessEqual(len(res), 1500000))
# can we replace files in place?
d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
"NEWER contents"))
b"NEWER contents"))
d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
d.addCallback(self.failUnlessEqual, "NEWER contents")
# test unlinked POST
d.addCallback(lambda res: self.POST("uri", t="upload",
file=("new.txt", "data" * 10000)))
d.addCallback(lambda res: self.POST("uri", t=b"upload",
file=("new.txt", b"data" * 10000)))
# and again using the helper, which exercises different upload-status
# display code
d.addCallback(lambda res: self.POST("uri", use_helper=True, t="upload",
file=("foo.txt", "data2" * 10000)))
d.addCallback(lambda res: self.POST("uri", use_helper=True, t=b"upload",
file=("foo.txt", b"data2" * 10000)))
# check that the status page exists
d.addCallback(lambda res: self.GET("status"))
@ -2105,7 +2137,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# exercise some of the diagnostic tools in runner.py
# find a share
for (dirpath, dirnames, filenames) in os.walk(unicode(self.basedir)):
for (dirpath, dirnames, filenames) in os.walk(ensure_text(self.basedir)):
if "storage" not in dirpath:
continue
if not filenames:
@ -2119,7 +2151,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
filename = os.path.join(dirpath, filenames[0])
# peek at the magic to see if it is a chk share
magic = open(filename, "rb").read(4)
if magic == '\x00\x00\x00\x01':
if magic == b'\x00\x00\x00\x01':
break
else:
self.fail("unable to find any uri_extension files in %r"
@ -2152,7 +2184,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# 'find-shares' tool
sharedir, shnum = os.path.split(filename)
storagedir, storage_index_s = os.path.split(sharedir)
storage_index_s = str(storage_index_s)
nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
rc,out,err = yield run_cli("debug", "find-shares", storage_index_s,
*nodedirs)
@ -2176,7 +2207,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# allmydata.control (mostly used for performance tests)
c0 = self.clients[0]
control_furl_file = c0.config.get_private_path("control.furl")
control_furl = open(control_furl_file, "r").read().strip()
control_furl = ensure_str(open(control_furl_file, "r").read().strip())
# it doesn't really matter which Tub we use to connect to the client,
# so let's just use our IntroducerNode's
d = self.introducer.tub.getReference(control_furl)
@ -2208,7 +2239,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# sure that works, before we add other aliases.
root_file = os.path.join(client0_basedir, "private", "root_dir.cap")
f = open(root_file, "w")
f = open(root_file, "wb")
f.write(private_uri)
f.close()
@ -2551,6 +2582,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
@skipIf(PY3, "Python 3 CLI support hasn't happened yet.")
def test_filesystem_with_cli_in_subprocess(self):
# We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
@ -2574,12 +2606,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
out, err, rc_or_sig = res
self.failUnlessEqual(rc_or_sig, 0, str(res))
if check_stderr:
self.failUnlessEqual(err, "")
self.failUnlessEqual(err, b"")
d.addCallback(_run_in_subprocess, "create-alias", "newalias")
d.addCallback(_check_succeeded)
STDIN_DATA = "This is the file to upload from stdin."
STDIN_DATA = b"This is the file to upload from stdin."
d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
d.addCallback(_check_succeeded, check_stderr=False)
@ -2601,7 +2633,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
def _test_checker(self, res):
ut = upload.Data("too big to be literal" * 200, convergence=None)
ut = upload.Data(b"too big to be literal" * 200, convergence=None)
d = self._personal_node.add_file(u"big file", ut)
d.addCallback(lambda res: self._personal_node.check(Monitor()))

4
src/allmydata/test/test_util.py
View File

@ -33,7 +33,9 @@ if six.PY3:
class IDLib(unittest.TestCase):
def test_nodeid_b2a(self):
self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
result = idlib.nodeid_b2a(b"\x00"*20)
self.assertEqual(result, "a"*32)
self.assertIsInstance(result, str)
class MyList(list):

3
src/allmydata/test/web/common.py
View File

@ -25,7 +25,8 @@ def assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs):
tags = soup.find_all(tag_name)
for tag in tags:
if all(v in tag.attrs.get(k, []) for k, v in attrs.items()):
return # we found every attr in this tag; done
# we found every attr in this tag; done
return tag
testcase.fail(
u"No <{}> tags contain attributes: {}".format(tag_name, attrs)
)

40
src/allmydata/test/web/test_root.py
View File

@ -1,7 +1,13 @@
from mock import Mock
import time
from urllib import (
quote,
)
from bs4 import (
BeautifulSoup,
)
from twisted.trial import unittest
from twisted.web.template import Tag
from twisted.web.test.requesthelper import DummyRequest
@ -16,6 +22,9 @@ from ...util.connection_status import ConnectionStatus
from allmydata.web.root import URIHandler
from allmydata.client import _Client
from .common import (
assert_soup_has_tag_with_attributes,
)
from ..common_web import (
render,
)
@ -30,28 +39,37 @@ class RenderSlashUri(unittest.TestCase):
"""
def setUp(self):
self.client = Mock()
self.client = object()
self.res = URIHandler(self.client)
def test_valid(self):
def test_valid_query_redirect(self):
"""
A valid capbility does not result in error
A syntactically valid capability given in the ``uri`` query argument
results in a redirect.
"""
query_args = {b"uri": [
cap = (
b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:"
b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882"
]}
)
query_args = {b"uri": [cap]}
response_body = self.successResultOf(
render(self.res, query_args),
)
self.assertNotEqual(
response_body,
"Invalid capability",
soup = BeautifulSoup(response_body, 'html5lib')
tag = assert_soup_has_tag_with_attributes(
self,
soup,
u"meta",
{u"http-equiv": "refresh"},
)
self.assertIn(
quote(cap, safe=""),
tag.attrs.get(u"content"),
)
def test_invalid(self):
"""
A (trivially) invalid capbility is an error
A syntactically invalid capbility results in an error.
"""
query_args = {b"uri": [b"not a capability"]}
response_body = self.successResultOf(

7
src/allmydata/util/_python3.py
View File

@ -97,6 +97,7 @@ PORTED_MODULES = [
"allmydata.util.happinessutil",
"allmydata.util.hashutil",
"allmydata.util.humanreadable",
"allmydata.util.idlib",
"allmydata.util.iputil",
"allmydata.util.jsonbytes",
"allmydata.util.log",
@ -168,6 +169,12 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_storage",
"allmydata.test.test_storage_client",
"allmydata.test.test_storage_web",
# Only partially ported, test_filesystem_with_cli_in_subprocess and
# test_filesystem methods aren't ported yet, should be done once CLI and
# web are ported respectively.
"allmydata.test.test_system",
"allmydata.test.test_time_format",
"allmydata.test.test_upload",
"allmydata.test.test_uri",

3
src/allmydata/util/base32.py
View File

@ -133,6 +133,9 @@ def a2b(cs):
"""
@param cs the base-32 encoded data (as bytes)
"""
# Workaround Future newbytes issues by converting to real bytes on Python 2:
if hasattr(cs, "__native__"):
cs = cs.__native__()
precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
precondition(isinstance(cs, bytes), cs)

24
src/allmydata/util/idlib.py
View File

@ -1,9 +1,29 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_text
from foolscap import base32
def nodeid_b2a(nodeid):
# we display nodeids using the same base32 alphabet that Foolscap uses
return base32.encode(nodeid)
"""
We display nodeids using the same base32 alphabet that Foolscap uses.
Returns a Unicode string.
"""
return ensure_text(base32.encode(nodeid))
def shortnodeid_b2a(nodeid):
"""
Short version of nodeid_b2a() output, Unicode string.
"""
return nodeid_b2a(nodeid)[:8]