mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-20 05:28:04 +00:00
Merge remote-tracking branch 'origin/3552.test_system-python-3' into 3565.web-tests-python-3-part-1
This commit is contained in:
Itamar Turner-Trauring
commit
9b59e7e245
0
newsfragments/3533.minor
Normal file
0
newsfragments/3533.minor
Normal file
0
newsfragments/3552.minor
Normal file
0
newsfragments/3552.minor
Normal file
@ -23,17 +23,12 @@ python.pkgs.buildPythonPackage rec {
|
||||
# This list is over-zealous because it's more work to disable individual
|
||||
# tests with in a module.
|
||||
|
||||
# test_system is a lot of integration-style tests that do a lot of real
|
||||
# networking between many processes. They sometimes fail spuriously.
|
||||
rm src/allmydata/test/test_system.py
|
||||
|
||||
# Many of these tests don't properly skip when i2p or tor dependencies are
|
||||
# not supplied (and we are not supplying them).
|
||||
rm src/allmydata/test/test_i2p_provider.py
|
||||
rm src/allmydata/test/test_connections.py
|
||||
rm src/allmydata/test/cli/test_create.py
|
||||
rm src/allmydata/test/test_client.py
|
||||
rm src/allmydata/test/test_runner.py
|
||||
'';
|
||||
|
||||
|
||||
|
||||
@ -714,7 +714,7 @@ class _Client(node.Node, pollmixin.PollMixin):
|
||||
def get_long_nodeid(self):
|
||||
# this matches what IServer.get_longname() says about us elsewhere
|
||||
vk_string = ed25519.string_from_verifying_key(self._node_public_key)
|
||||
return remove_prefix(vk_string, "pub-")
|
||||
return remove_prefix(vk_string, b"pub-")
|
||||
|
||||
def get_long_tubid(self):
|
||||
return idlib.nodeid_b2a(self.nodeid)
|
||||
@ -898,10 +898,6 @@ class _Client(node.Node, pollmixin.PollMixin):
|
||||
if helper_furl in ("None", ""):
|
||||
helper_furl = None
|
||||
|
||||
# FURLs need to be bytes:
|
||||
if helper_furl is not None:
|
||||
helper_furl = helper_furl.encode("utf-8")
|
||||
|
||||
DEP = self.encoding_params
|
||||
DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
|
||||
DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))
|
||||
|
||||
@ -9,6 +9,7 @@ from __future__ import unicode_literals
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from six import ensure_str
|
||||
|
||||
import time
|
||||
now = time.time
|
||||
@ -98,7 +99,7 @@ class ShareFinder(object):
|
||||
|
||||
# internal methods
|
||||
def loop(self):
|
||||
pending_s = ",".join([rt.server.get_name()
|
||||
pending_s = ",".join([ensure_str(rt.server.get_name())
|
||||
for rt in self.pending_requests]) # sort?
|
||||
self.log(format="ShareFinder loop: running=%(running)s"
|
||||
" hungry=%(hungry)s, pending=%(pending)s",
|
||||
|
||||
@ -11,6 +11,7 @@ from future.utils import PY2, native_str
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from past.builtins import long, unicode
|
||||
from six import ensure_str
|
||||
|
||||
import os, time, weakref, itertools
|
||||
from zope.interface import implementer
|
||||
@ -1825,7 +1826,7 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
|
||||
def startService(self):
|
||||
service.MultiService.startService(self)
|
||||
if self._helper_furl:
|
||||
self.parent.tub.connectTo(self._helper_furl,
|
||||
self.parent.tub.connectTo(ensure_str(self._helper_furl),
|
||||
self._got_helper)
|
||||
|
||||
def _got_helper(self, helper):
|
||||
|
||||
@ -11,7 +11,7 @@ if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from past.builtins import long
|
||||
|
||||
from six import ensure_text
|
||||
from six import ensure_text, ensure_str
|
||||
|
||||
import time
|
||||
from zope.interface import implementer
|
||||
@ -39,8 +39,6 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
nickname, my_version, oldest_supported,
|
||||
sequencer, cache_filepath):
|
||||
self._tub = tub
|
||||
if isinstance(introducer_furl, str):
|
||||
introducer_furl = introducer_furl.encode("utf-8")
|
||||
self.introducer_furl = introducer_furl
|
||||
|
||||
assert isinstance(nickname, str)
|
||||
@ -96,7 +94,7 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
def startService(self):
|
||||
service.Service.startService(self)
|
||||
self._introducer_error = None
|
||||
rc = self._tub.connectTo(self.introducer_furl, self._got_introducer)
|
||||
rc = self._tub.connectTo(ensure_str(self.introducer_furl), self._got_introducer)
|
||||
self._introducer_reconnector = rc
|
||||
def connect_failed(failure):
|
||||
self.log("Initial Introducer connection failed: perhaps it's down",
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
from __future__ import print_function
|
||||
|
||||
from future.utils import PY2, native_str
|
||||
from future.builtins import str as future_str
|
||||
|
||||
import os
|
||||
import time
|
||||
import signal
|
||||
@ -48,24 +51,23 @@ def _getvalue(io):
|
||||
return io.read()
|
||||
|
||||
|
||||
def run_cli_bytes(verb, *args, **kwargs):
|
||||
def run_cli_native(verb, *args, **kwargs):
|
||||
"""
|
||||
Run a Tahoe-LAFS CLI command specified as bytes.
|
||||
Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode
|
||||
(on Python 3); basically, it accepts a native string.
|
||||
|
||||
Most code should prefer ``run_cli_unicode`` which deals with all the
|
||||
necessary encoding considerations. This helper still exists so that novel
|
||||
misconfigurations can be explicitly tested (for example, receiving UTF-8
|
||||
bytes when the system encoding claims to be ASCII).
|
||||
necessary encoding considerations.
|
||||
|
||||
:param bytes verb: The command to run. For example, ``b"create-node"``.
|
||||
:param native_str verb: The command to run. For example, ``b"create-node"``.
|
||||
|
||||
:param [bytes] args: The arguments to pass to the command. For example,
|
||||
:param [native_str] args: The arguments to pass to the command. For example,
|
||||
``(b"--hostname=localhost",)``.
|
||||
|
||||
:param [bytes] nodeargs: Extra arguments to pass to the Tahoe executable
|
||||
:param [native_str] nodeargs: Extra arguments to pass to the Tahoe executable
|
||||
before ``verb``.
|
||||
|
||||
:param bytes stdin: Text to pass to the command via stdin.
|
||||
:param native_str stdin: Text to pass to the command via stdin.
|
||||
|
||||
:param NoneType|str encoding: The name of an encoding which stdout and
|
||||
stderr will be configured to use. ``None`` means stdout and stderr
|
||||
@ -75,8 +77,8 @@ def run_cli_bytes(verb, *args, **kwargs):
|
||||
nodeargs = kwargs.pop("nodeargs", [])
|
||||
encoding = kwargs.pop("encoding", None)
|
||||
precondition(
|
||||
all(isinstance(arg, bytes) for arg in [verb] + nodeargs + list(args)),
|
||||
"arguments to run_cli must be bytes -- convert using unicode_to_argv",
|
||||
all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)),
|
||||
"arguments to run_cli must be a native string -- convert using unicode_to_argv",
|
||||
verb=verb,
|
||||
args=args,
|
||||
nodeargs=nodeargs,
|
||||
@ -135,15 +137,19 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
|
||||
if nodeargs is None:
|
||||
nodeargs = []
|
||||
precondition(
|
||||
all(isinstance(arg, unicode) for arg in [verb] + nodeargs + argv),
|
||||
all(isinstance(arg, future_str) for arg in [verb] + nodeargs + argv),
|
||||
"arguments to run_cli_unicode must be unicode",
|
||||
verb=verb,
|
||||
nodeargs=nodeargs,
|
||||
argv=argv,
|
||||
)
|
||||
codec = encoding or "ascii"
|
||||
encode = lambda t: None if t is None else t.encode(codec)
|
||||
d = run_cli_bytes(
|
||||
if PY2:
|
||||
encode = lambda t: None if t is None else t.encode(codec)
|
||||
else:
|
||||
# On Python 3 command-line parsing expects Unicode!
|
||||
encode = lambda t: t
|
||||
d = run_cli_native(
|
||||
encode(verb),
|
||||
nodeargs=list(encode(arg) for arg in nodeargs),
|
||||
stdin=encode(stdin),
|
||||
@ -161,7 +167,7 @@ def run_cli_unicode(verb, argv, nodeargs=None, stdin=None, encoding=None):
|
||||
return d
|
||||
|
||||
|
||||
run_cli = run_cli_bytes
|
||||
run_cli = run_cli_native
|
||||
|
||||
|
||||
def parse_cli(*argv):
|
||||
|
||||
@ -47,12 +47,17 @@ class VerboseError(Error):
|
||||
|
||||
@inlineCallbacks
|
||||
def do_http(method, url, **kwargs):
|
||||
"""
|
||||
Run HTTP query, return Deferred of body as bytes.
|
||||
"""
|
||||
response = yield treq.request(method, url, persistent=False, **kwargs)
|
||||
body = yield treq.content(response)
|
||||
# TODO: replace this with response.fail_for_status when
|
||||
# https://github.com/twisted/treq/pull/159 has landed
|
||||
if 400 <= response.code < 600:
|
||||
raise VerboseError(response.code, response=body)
|
||||
raise VerboseError(
|
||||
response.code, response="For request {} to {}, got: {}".format(
|
||||
method, url, body))
|
||||
returnValue(body)
|
||||
|
||||
|
||||
|
||||
@ -1,7 +1,22 @@
|
||||
"""
|
||||
Ported to Python 3, partially: test_filesystem* will be done in a future round.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2, PY3
|
||||
if PY2:
|
||||
# Don't import bytes since it causes issues on (so far unported) modules on Python 2.
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401
|
||||
|
||||
from past.builtins import chr as byteschr, long
|
||||
from six import ensure_text, ensure_str
|
||||
|
||||
import os, re, sys, time, json
|
||||
from functools import partial
|
||||
from unittest import skipIf
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
@ -40,7 +55,7 @@ from .common import (
|
||||
TEST_RSA_KEY_SIZE,
|
||||
SameProcessStreamEndpointAssigner,
|
||||
)
|
||||
from .common_web import do_http, Error
|
||||
from .common_web import do_http as do_http_bytes, Error
|
||||
from .web.common import (
|
||||
assert_soup_has_tag_with_attributes
|
||||
)
|
||||
@ -48,12 +63,28 @@ from .web.common import (
|
||||
# TODO: move this to common or common_util
|
||||
from allmydata.test.test_runner import RunBinTahoeMixin
|
||||
from . import common_util as testutil
|
||||
from .common_util import run_cli
|
||||
from .common_util import run_cli_unicode
|
||||
from ..scripts.common import (
|
||||
write_introducer,
|
||||
)
|
||||
|
||||
LARGE_DATA = """
|
||||
def run_cli(*args, **kwargs):
|
||||
"""
|
||||
Backwards compatible version so we don't have to change all the tests.
|
||||
"""
|
||||
nodeargs = [ensure_text(a) for a in kwargs.pop("nodeargs", [])]
|
||||
kwargs["nodeargs"] = nodeargs
|
||||
return run_cli_unicode(
|
||||
ensure_text(args[0]), [ensure_text(a) for a in args[1:]], **kwargs)
|
||||
|
||||
|
||||
def do_http(*args, **kwargs):
|
||||
"""Wrapper for do_http() that returns Unicode."""
|
||||
return do_http_bytes(*args, **kwargs).addCallback(
|
||||
lambda b: str(b, "utf-8"))
|
||||
|
||||
|
||||
LARGE_DATA = b"""
|
||||
This is some data to publish to the remote grid.., which needs to be large
|
||||
enough to not fit inside a LIT uri.
|
||||
"""
|
||||
@ -629,7 +660,7 @@ def _render_config(config):
|
||||
"""
|
||||
Convert a ``dict`` of ``dict`` of ``bytes`` to an ini-format string.
|
||||
"""
|
||||
return "\n\n".join(list(
|
||||
return u"\n\n".join(list(
|
||||
_render_config_section(k, v)
|
||||
for (k, v)
|
||||
in config.items()
|
||||
@ -640,7 +671,7 @@ def _render_config_section(heading, values):
|
||||
Convert a ``bytes`` heading and a ``dict`` of ``bytes`` to an ini-format
|
||||
section as ``bytes``.
|
||||
"""
|
||||
return "[{}]\n{}\n".format(
|
||||
return u"[{}]\n{}\n".format(
|
||||
heading, _render_section_values(values)
|
||||
)
|
||||
|
||||
@ -649,8 +680,8 @@ def _render_section_values(values):
|
||||
Convert a ``dict`` of ``bytes`` to the body of an ini-format section as
|
||||
``bytes``.
|
||||
"""
|
||||
return "\n".join(list(
|
||||
"{} = {}".format(k, v)
|
||||
return u"\n".join(list(
|
||||
u"{} = {}".format(k, v)
|
||||
for (k, v)
|
||||
in sorted(values.items())
|
||||
))
|
||||
@ -753,7 +784,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
|
||||
self.helper_furl = helper_furl
|
||||
if self.numclients >= 4:
|
||||
with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+') as f:
|
||||
with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f:
|
||||
f.write(
|
||||
"[client]\n"
|
||||
"helper.furl = {}\n".format(helper_furl)
|
||||
@ -796,8 +827,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
|
||||
def setconf(config, which, section, feature, value):
|
||||
if which in feature_matrix.get((section, feature), {which}):
|
||||
if isinstance(value, unicode):
|
||||
value = value.encode("utf-8")
|
||||
config.setdefault(section, {})[feature] = value
|
||||
|
||||
setnode = partial(setconf, config, which, "node")
|
||||
@ -870,7 +899,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
config = "[client]\n"
|
||||
if helper_furl:
|
||||
config += "helper.furl = %s\n" % helper_furl
|
||||
basedir.child("tahoe.cfg").setContent(config)
|
||||
basedir.child("tahoe.cfg").setContent(config.encode("utf-8"))
|
||||
private = basedir.child("private")
|
||||
private.makedirs()
|
||||
write_introducer(
|
||||
@ -980,12 +1009,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
|
||||
def test_upload_and_download_convergent(self):
|
||||
self.basedir = "system/SystemTest/test_upload_and_download_convergent"
|
||||
return self._test_upload_and_download(convergence="some convergence string")
|
||||
return self._test_upload_and_download(convergence=b"some convergence string")
|
||||
|
||||
def _test_upload_and_download(self, convergence):
|
||||
# we use 4000 bytes of data, which will result in about 400k written
|
||||
# to disk among all our simulated nodes
|
||||
DATA = "Some data to upload\n" * 200
|
||||
DATA = b"Some data to upload\n" * 200
|
||||
d = self.set_up_nodes()
|
||||
def _check_connections(res):
|
||||
for c in self.clients:
|
||||
@ -993,7 +1022,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
all_peerids = c.get_storage_broker().get_all_serverids()
|
||||
self.failUnlessEqual(len(all_peerids), self.numclients)
|
||||
sb = c.storage_broker
|
||||
permuted_peers = sb.get_servers_for_psi("a")
|
||||
permuted_peers = sb.get_servers_for_psi(b"a")
|
||||
self.failUnlessEqual(len(permuted_peers), self.numclients)
|
||||
d.addCallback(_check_connections)
|
||||
|
||||
@ -1016,7 +1045,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
theuri = results.get_uri()
|
||||
log.msg("upload finished: uri is %s" % (theuri,))
|
||||
self.uri = theuri
|
||||
assert isinstance(self.uri, str), self.uri
|
||||
assert isinstance(self.uri, bytes), self.uri
|
||||
self.cap = uri.from_string(self.uri)
|
||||
self.n = self.clients[1].create_node_from_uri(self.uri)
|
||||
d.addCallback(_upload_done)
|
||||
@ -1050,17 +1079,17 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
d.addCallback(lambda ign:
|
||||
n.read(MemoryConsumer(), offset=1, size=4))
|
||||
def _read_portion_done(mc):
|
||||
self.failUnlessEqual("".join(mc.chunks), DATA[1:1+4])
|
||||
self.failUnlessEqual(b"".join(mc.chunks), DATA[1:1+4])
|
||||
d.addCallback(_read_portion_done)
|
||||
d.addCallback(lambda ign:
|
||||
n.read(MemoryConsumer(), offset=2, size=None))
|
||||
def _read_tail_done(mc):
|
||||
self.failUnlessEqual("".join(mc.chunks), DATA[2:])
|
||||
self.failUnlessEqual(b"".join(mc.chunks), DATA[2:])
|
||||
d.addCallback(_read_tail_done)
|
||||
d.addCallback(lambda ign:
|
||||
n.read(MemoryConsumer(), size=len(DATA)+1000))
|
||||
def _read_too_much(mc):
|
||||
self.failUnlessEqual("".join(mc.chunks), DATA)
|
||||
self.failUnlessEqual(b"".join(mc.chunks), DATA)
|
||||
d.addCallback(_read_too_much)
|
||||
|
||||
return d
|
||||
@ -1110,7 +1139,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return connected
|
||||
d.addCallback(lambda ign: self.poll(_has_helper))
|
||||
|
||||
HELPER_DATA = "Data that needs help to upload" * 1000
|
||||
HELPER_DATA = b"Data that needs help to upload" * 1000
|
||||
def _upload_with_helper(res):
|
||||
u = upload.Data(HELPER_DATA, convergence=convergence)
|
||||
d = self.extra_node.upload(u)
|
||||
@ -1144,7 +1173,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
d.addCallback(fireEventually)
|
||||
|
||||
def _upload_resumable(res):
|
||||
DATA = "Data that needs help to upload and gets interrupted" * 1000
|
||||
DATA = b"Data that needs help to upload and gets interrupted" * 1000
|
||||
u1 = CountingDataUploadable(DATA, convergence=convergence)
|
||||
u2 = CountingDataUploadable(DATA, convergence=convergence)
|
||||
|
||||
@ -1266,7 +1295,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
s = stats["stats"]
|
||||
self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1)
|
||||
c = stats["counters"]
|
||||
self.failUnless("storage_server.allocate" in c)
|
||||
# Probably this should be Unicode eventually? But we haven't ported
|
||||
# stats code yet.
|
||||
self.failUnless(b"storage_server.allocate" in c)
|
||||
d.addCallback(_grab_stats)
|
||||
|
||||
return d
|
||||
@ -1287,7 +1318,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
assert pieces[-5].startswith("client")
|
||||
client_num = int(pieces[-5][-1])
|
||||
storage_index_s = pieces[-1]
|
||||
storage_index = si_a2b(storage_index_s)
|
||||
storage_index = si_a2b(storage_index_s.encode("ascii"))
|
||||
for sharename in filenames:
|
||||
shnum = int(sharename)
|
||||
filename = os.path.join(dirpath, sharename)
|
||||
@ -1320,7 +1351,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
elif which == "signature":
|
||||
signature = self.flip_bit(signature)
|
||||
elif which == "share_hash_chain":
|
||||
nodenum = share_hash_chain.keys()[0]
|
||||
nodenum = list(share_hash_chain.keys())[0]
|
||||
share_hash_chain[nodenum] = self.flip_bit(share_hash_chain[nodenum])
|
||||
elif which == "block_hash_tree":
|
||||
block_hash_tree[-1] = self.flip_bit(block_hash_tree[-1])
|
||||
@ -1343,11 +1374,11 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
|
||||
def test_mutable(self):
|
||||
self.basedir = "system/SystemTest/test_mutable"
|
||||
DATA = "initial contents go here." # 25 bytes % 3 != 0
|
||||
DATA = b"initial contents go here." # 25 bytes % 3 != 0
|
||||
DATA_uploadable = MutableData(DATA)
|
||||
NEWDATA = "new contents yay"
|
||||
NEWDATA = b"new contents yay"
|
||||
NEWDATA_uploadable = MutableData(NEWDATA)
|
||||
NEWERDATA = "this is getting old"
|
||||
NEWERDATA = b"this is getting old"
|
||||
NEWERDATA_uploadable = MutableData(NEWERDATA)
|
||||
|
||||
d = self.set_up_nodes()
|
||||
@ -1396,7 +1427,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
self.failUnless(" share_hash_chain: " in output)
|
||||
self.failUnless(" block_hash_tree: 1 nodes\n" in output)
|
||||
expected = (" verify-cap: URI:SSK-Verifier:%s:" %
|
||||
base32.b2a(storage_index))
|
||||
str(base32.b2a(storage_index), "ascii"))
|
||||
self.failUnless(expected in output)
|
||||
except unittest.FailTest:
|
||||
print()
|
||||
@ -1475,7 +1506,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
for (client_num, storage_index, filename, shnum)
|
||||
in shares ])
|
||||
assert len(where) == 10 # this test is designed for 3-of-10
|
||||
for shnum, filename in where.items():
|
||||
for shnum, filename in list(where.items()):
|
||||
# shares 7,8,9 are left alone. read will check
|
||||
# (share_hash_chain, block_hash_tree, share_data). New
|
||||
# seqnum+R pairs will trigger a check of (seqnum, R, IV,
|
||||
@ -1525,9 +1556,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
def _check_empty_file(res):
|
||||
# make sure we can create empty files, this usually screws up the
|
||||
# segsize math
|
||||
d1 = self.clients[2].create_mutable_file(MutableData(""))
|
||||
d1 = self.clients[2].create_mutable_file(MutableData(b""))
|
||||
d1.addCallback(lambda newnode: newnode.download_best_version())
|
||||
d1.addCallback(lambda res: self.failUnlessEqual("", res))
|
||||
d1.addCallback(lambda res: self.failUnlessEqual(b"", res))
|
||||
return d1
|
||||
d.addCallback(_check_empty_file)
|
||||
|
||||
@ -1550,7 +1581,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return d
|
||||
|
||||
def flip_bit(self, good):
|
||||
return good[:-1] + chr(ord(good[-1]) ^ 0x01)
|
||||
return good[:-1] + byteschr(ord(good[-1:]) ^ 0x01)
|
||||
|
||||
def mangle_uri(self, gooduri):
|
||||
# change the key, which changes the storage index, which means we'll
|
||||
@ -1571,6 +1602,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# the key, which should cause the download to fail the post-download
|
||||
# plaintext_hash check.
|
||||
|
||||
@skipIf(PY3, "Python 3 web support hasn't happened yet.")
|
||||
def test_filesystem(self):
|
||||
self.basedir = "system/SystemTest/test_filesystem"
|
||||
self.data = LARGE_DATA
|
||||
@ -1632,7 +1664,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
d1.addCallback(self.log, "publish finished")
|
||||
def _stash_uri(filenode):
|
||||
self.uri = filenode.get_uri()
|
||||
assert isinstance(self.uri, str), (self.uri, filenode)
|
||||
assert isinstance(self.uri, bytes), (self.uri, filenode)
|
||||
d1.addCallback(_stash_uri)
|
||||
return d1
|
||||
d.addCallback(_made_subdir1)
|
||||
@ -1650,7 +1682,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return res
|
||||
|
||||
def _do_publish_private(self, res):
|
||||
self.smalldata = "sssh, very secret stuff"
|
||||
self.smalldata = b"sssh, very secret stuff"
|
||||
ut = upload.Data(self.smalldata, convergence=None)
|
||||
d = self.clients[0].create_dirnode()
|
||||
d.addCallback(self.log, "GOT private directory")
|
||||
@ -1737,7 +1769,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "mkdir(nope)", None, dirnode.create_subdirectory, u"nope"))
|
||||
|
||||
d1.addCallback(self.log, "doing add_file(ro)")
|
||||
ut = upload.Data("I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence="99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
|
||||
ut = upload.Data(b"I will disappear, unrecorded and unobserved. The tragedy of my demise is made more poignant by its silence, but this beauty is not for you to ever know.", convergence=b"99i-p1x4-xd4-18yc-ywt-87uu-msu-zo -- completely and totally unguessable string (unless you read this)")
|
||||
d1.addCallback(lambda res: self.shouldFail2(NotWriteableError, "add_file(nope)", None, dirnode.add_file, u"hope", ut))
|
||||
|
||||
d1.addCallback(self.log, "doing get(ro)")
|
||||
@ -1801,7 +1833,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
"largest-directory-children": 3,
|
||||
"largest-immutable-file": 112,
|
||||
}
|
||||
for k,v in expected.iteritems():
|
||||
for k,v in list(expected.items()):
|
||||
self.failUnlessEqual(stats[k], v,
|
||||
"stats[%s] was %s, not %s" %
|
||||
(k, stats[k], v))
|
||||
@ -1850,33 +1882,33 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return do_http("get", self.webish_url + urlpath)
|
||||
|
||||
def POST(self, urlpath, use_helper=False, **fields):
|
||||
sepbase = "boogabooga"
|
||||
sep = "--" + sepbase
|
||||
sepbase = b"boogabooga"
|
||||
sep = b"--" + sepbase
|
||||
form = []
|
||||
form.append(sep)
|
||||
form.append('Content-Disposition: form-data; name="_charset"')
|
||||
form.append('')
|
||||
form.append('UTF-8')
|
||||
form.append(b'Content-Disposition: form-data; name="_charset"')
|
||||
form.append(b'')
|
||||
form.append(b'UTF-8')
|
||||
form.append(sep)
|
||||
for name, value in fields.iteritems():
|
||||
for name, value in fields.items():
|
||||
if isinstance(value, tuple):
|
||||
filename, value = value
|
||||
form.append('Content-Disposition: form-data; name="%s"; '
|
||||
'filename="%s"' % (name, filename.encode("utf-8")))
|
||||
form.append(b'Content-Disposition: form-data; name="%s"; '
|
||||
b'filename="%s"' % (name, filename.encode("utf-8")))
|
||||
else:
|
||||
form.append('Content-Disposition: form-data; name="%s"' % name)
|
||||
form.append('')
|
||||
form.append(str(value))
|
||||
form.append(b'Content-Disposition: form-data; name="%s"' % name)
|
||||
form.append(b'')
|
||||
form.append(b"%s" % (value,))
|
||||
form.append(sep)
|
||||
form[-1] += "--"
|
||||
body = ""
|
||||
form[-1] += b"--"
|
||||
body = b""
|
||||
headers = {}
|
||||
if fields:
|
||||
body = "\r\n".join(form) + "\r\n"
|
||||
headers["content-type"] = "multipart/form-data; boundary=%s" % sepbase
|
||||
body = b"\r\n".join(form) + b"\r\n"
|
||||
headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii")
|
||||
return self.POST2(urlpath, body, headers, use_helper)
|
||||
|
||||
def POST2(self, urlpath, body="", headers={}, use_helper=False):
|
||||
def POST2(self, urlpath, body=b"", headers={}, use_helper=False):
|
||||
if use_helper:
|
||||
url = self.helper_webish_url + urlpath
|
||||
else:
|
||||
@ -1884,7 +1916,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return do_http("post", url, data=body, headers=headers)
|
||||
|
||||
def _test_web(self, res):
|
||||
public = "uri/" + self._root_directory_uri
|
||||
public = "uri/" + str(self._root_directory_uri, "ascii")
|
||||
d = self.GET("")
|
||||
def _got_welcome(page):
|
||||
html = page.replace('\n', ' ')
|
||||
@ -1893,7 +1925,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
"I didn't see the right '%s' message in:\n%s" % (connected_re, page))
|
||||
# nodeids/tubids don't have any regexp-special characters
|
||||
nodeid_re = r'<th>Node ID:</th>\s*<td title="TubID: %s">%s</td>' % (
|
||||
self.clients[0].get_long_tubid(), self.clients[0].get_long_nodeid())
|
||||
self.clients[0].get_long_tubid(), str(self.clients[0].get_long_nodeid(), "ascii"))
|
||||
self.failUnless(re.search(nodeid_re, html),
|
||||
"I didn't see the right '%s' message in:\n%s" % (nodeid_re, page))
|
||||
self.failUnless("Helper: 0 active uploads" in page)
|
||||
@ -1954,7 +1986,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# upload a file with PUT
|
||||
d.addCallback(self.log, "about to try PUT")
|
||||
d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
|
||||
"new.txt contents"))
|
||||
b"new.txt contents"))
|
||||
d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
|
||||
d.addCallback(self.failUnlessEqual, "new.txt contents")
|
||||
# and again with something large enough to use multiple segments,
|
||||
@ -1965,23 +1997,23 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
c.encoding_params['happy'] = 1
|
||||
d.addCallback(_new_happy_semantics)
|
||||
d.addCallback(lambda res: self.PUT(public + "/subdir3/big.txt",
|
||||
"big" * 500000)) # 1.5MB
|
||||
b"big" * 500000)) # 1.5MB
|
||||
d.addCallback(lambda res: self.GET(public + "/subdir3/big.txt"))
|
||||
d.addCallback(lambda res: self.failUnlessEqual(len(res), 1500000))
|
||||
|
||||
# can we replace files in place?
|
||||
d.addCallback(lambda res: self.PUT(public + "/subdir3/new.txt",
|
||||
"NEWER contents"))
|
||||
b"NEWER contents"))
|
||||
d.addCallback(lambda res: self.GET(public + "/subdir3/new.txt"))
|
||||
d.addCallback(self.failUnlessEqual, "NEWER contents")
|
||||
|
||||
# test unlinked POST
|
||||
d.addCallback(lambda res: self.POST("uri", t="upload",
|
||||
file=("new.txt", "data" * 10000)))
|
||||
d.addCallback(lambda res: self.POST("uri", t=b"upload",
|
||||
file=("new.txt", b"data" * 10000)))
|
||||
# and again using the helper, which exercises different upload-status
|
||||
# display code
|
||||
d.addCallback(lambda res: self.POST("uri", use_helper=True, t="upload",
|
||||
file=("foo.txt", "data2" * 10000)))
|
||||
d.addCallback(lambda res: self.POST("uri", use_helper=True, t=b"upload",
|
||||
file=("foo.txt", b"data2" * 10000)))
|
||||
|
||||
# check that the status page exists
|
||||
d.addCallback(lambda res: self.GET("status"))
|
||||
@ -2105,7 +2137,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# exercise some of the diagnostic tools in runner.py
|
||||
|
||||
# find a share
|
||||
for (dirpath, dirnames, filenames) in os.walk(unicode(self.basedir)):
|
||||
for (dirpath, dirnames, filenames) in os.walk(ensure_text(self.basedir)):
|
||||
if "storage" not in dirpath:
|
||||
continue
|
||||
if not filenames:
|
||||
@ -2119,7 +2151,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
filename = os.path.join(dirpath, filenames[0])
|
||||
# peek at the magic to see if it is a chk share
|
||||
magic = open(filename, "rb").read(4)
|
||||
if magic == '\x00\x00\x00\x01':
|
||||
if magic == b'\x00\x00\x00\x01':
|
||||
break
|
||||
else:
|
||||
self.fail("unable to find any uri_extension files in %r"
|
||||
@ -2152,7 +2184,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# 'find-shares' tool
|
||||
sharedir, shnum = os.path.split(filename)
|
||||
storagedir, storage_index_s = os.path.split(sharedir)
|
||||
storage_index_s = str(storage_index_s)
|
||||
nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
|
||||
rc,out,err = yield run_cli("debug", "find-shares", storage_index_s,
|
||||
*nodedirs)
|
||||
@ -2176,7 +2207,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# allmydata.control (mostly used for performance tests)
|
||||
c0 = self.clients[0]
|
||||
control_furl_file = c0.config.get_private_path("control.furl")
|
||||
control_furl = open(control_furl_file, "r").read().strip()
|
||||
control_furl = ensure_str(open(control_furl_file, "r").read().strip())
|
||||
# it doesn't really matter which Tub we use to connect to the client,
|
||||
# so let's just use our IntroducerNode's
|
||||
d = self.introducer.tub.getReference(control_furl)
|
||||
@ -2208,7 +2239,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
# sure that works, before we add other aliases.
|
||||
|
||||
root_file = os.path.join(client0_basedir, "private", "root_dir.cap")
|
||||
f = open(root_file, "w")
|
||||
f = open(root_file, "wb")
|
||||
f.write(private_uri)
|
||||
f.close()
|
||||
|
||||
@ -2551,6 +2582,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
|
||||
return d
|
||||
|
||||
@skipIf(PY3, "Python 3 CLI support hasn't happened yet.")
|
||||
def test_filesystem_with_cli_in_subprocess(self):
|
||||
# We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
|
||||
|
||||
@ -2574,12 +2606,12 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
out, err, rc_or_sig = res
|
||||
self.failUnlessEqual(rc_or_sig, 0, str(res))
|
||||
if check_stderr:
|
||||
self.failUnlessEqual(err, "")
|
||||
self.failUnlessEqual(err, b"")
|
||||
|
||||
d.addCallback(_run_in_subprocess, "create-alias", "newalias")
|
||||
d.addCallback(_check_succeeded)
|
||||
|
||||
STDIN_DATA = "This is the file to upload from stdin."
|
||||
STDIN_DATA = b"This is the file to upload from stdin."
|
||||
d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA)
|
||||
d.addCallback(_check_succeeded, check_stderr=False)
|
||||
|
||||
@ -2601,7 +2633,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
|
||||
return d
|
||||
|
||||
def _test_checker(self, res):
|
||||
ut = upload.Data("too big to be literal" * 200, convergence=None)
|
||||
ut = upload.Data(b"too big to be literal" * 200, convergence=None)
|
||||
d = self._personal_node.add_file(u"big file", ut)
|
||||
|
||||
d.addCallback(lambda res: self._personal_node.check(Monitor()))
|
||||
|
||||
@ -33,7 +33,9 @@ if six.PY3:
|
||||
|
||||
class IDLib(unittest.TestCase):
|
||||
def test_nodeid_b2a(self):
|
||||
self.failUnlessEqual(idlib.nodeid_b2a(b"\x00"*20), "a"*32)
|
||||
result = idlib.nodeid_b2a(b"\x00"*20)
|
||||
self.assertEqual(result, "a"*32)
|
||||
self.assertIsInstance(result, str)
|
||||
|
||||
|
||||
class MyList(list):
|
||||
|
||||
@ -25,7 +25,8 @@ def assert_soup_has_tag_with_attributes(testcase, soup, tag_name, attrs):
|
||||
tags = soup.find_all(tag_name)
|
||||
for tag in tags:
|
||||
if all(v in tag.attrs.get(k, []) for k, v in attrs.items()):
|
||||
return # we found every attr in this tag; done
|
||||
# we found every attr in this tag; done
|
||||
return tag
|
||||
testcase.fail(
|
||||
u"No <{}> tags contain attributes: {}".format(tag_name, attrs)
|
||||
)
|
||||
|
||||
@ -1,7 +1,13 @@
|
||||
from mock import Mock
|
||||
|
||||
import time
|
||||
|
||||
from urllib import (
|
||||
quote,
|
||||
)
|
||||
|
||||
from bs4 import (
|
||||
BeautifulSoup,
|
||||
)
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.web.template import Tag
|
||||
from twisted.web.test.requesthelper import DummyRequest
|
||||
@ -16,6 +22,9 @@ from ...util.connection_status import ConnectionStatus
|
||||
from allmydata.web.root import URIHandler
|
||||
from allmydata.client import _Client
|
||||
|
||||
from .common import (
|
||||
assert_soup_has_tag_with_attributes,
|
||||
)
|
||||
from ..common_web import (
|
||||
render,
|
||||
)
|
||||
@ -30,28 +39,37 @@ class RenderSlashUri(unittest.TestCase):
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.client = Mock()
|
||||
self.client = object()
|
||||
self.res = URIHandler(self.client)
|
||||
|
||||
def test_valid(self):
|
||||
def test_valid_query_redirect(self):
|
||||
"""
|
||||
A valid capbility does not result in error
|
||||
A syntactically valid capability given in the ``uri`` query argument
|
||||
results in a redirect.
|
||||
"""
|
||||
query_args = {b"uri": [
|
||||
cap = (
|
||||
b"URI:CHK:nt2xxmrccp7sursd6yh2thhcky:"
|
||||
b"mukesarwdjxiyqsjinbfiiro6q7kgmmekocxfjcngh23oxwyxtzq:2:5:5874882"
|
||||
]}
|
||||
)
|
||||
query_args = {b"uri": [cap]}
|
||||
response_body = self.successResultOf(
|
||||
render(self.res, query_args),
|
||||
)
|
||||
self.assertNotEqual(
|
||||
response_body,
|
||||
"Invalid capability",
|
||||
soup = BeautifulSoup(response_body, 'html5lib')
|
||||
tag = assert_soup_has_tag_with_attributes(
|
||||
self,
|
||||
soup,
|
||||
u"meta",
|
||||
{u"http-equiv": "refresh"},
|
||||
)
|
||||
self.assertIn(
|
||||
quote(cap, safe=""),
|
||||
tag.attrs.get(u"content"),
|
||||
)
|
||||
|
||||
def test_invalid(self):
|
||||
"""
|
||||
A (trivially) invalid capbility is an error
|
||||
A syntactically invalid capbility results in an error.
|
||||
"""
|
||||
query_args = {b"uri": [b"not a capability"]}
|
||||
response_body = self.successResultOf(
|
||||
|
||||
@ -97,6 +97,7 @@ PORTED_MODULES = [
|
||||
"allmydata.util.happinessutil",
|
||||
"allmydata.util.hashutil",
|
||||
"allmydata.util.humanreadable",
|
||||
"allmydata.util.idlib",
|
||||
"allmydata.util.iputil",
|
||||
"allmydata.util.jsonbytes",
|
||||
"allmydata.util.log",
|
||||
@ -168,6 +169,12 @@ PORTED_TEST_MODULES = [
|
||||
"allmydata.test.test_storage",
|
||||
"allmydata.test.test_storage_client",
|
||||
"allmydata.test.test_storage_web",
|
||||
|
||||
# Only partially ported, test_filesystem_with_cli_in_subprocess and
|
||||
# test_filesystem methods aren't ported yet, should be done once CLI and
|
||||
# web are ported respectively.
|
||||
"allmydata.test.test_system",
|
||||
|
||||
"allmydata.test.test_time_format",
|
||||
"allmydata.test.test_upload",
|
||||
"allmydata.test.test_uri",
|
||||
|
||||
@ -133,6 +133,9 @@ def a2b(cs):
|
||||
"""
|
||||
@param cs the base-32 encoded data (as bytes)
|
||||
"""
|
||||
# Workaround Future newbytes issues by converting to real bytes on Python 2:
|
||||
if hasattr(cs, "__native__"):
|
||||
cs = cs.__native__()
|
||||
precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
|
||||
precondition(isinstance(cs, bytes), cs)
|
||||
|
||||
|
||||
@ -1,9 +1,29 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
from six import ensure_text
|
||||
from foolscap import base32
|
||||
|
||||
|
||||
def nodeid_b2a(nodeid):
|
||||
# we display nodeids using the same base32 alphabet that Foolscap uses
|
||||
return base32.encode(nodeid)
|
||||
"""
|
||||
We display nodeids using the same base32 alphabet that Foolscap uses.
|
||||
|
||||
Returns a Unicode string.
|
||||
"""
|
||||
return ensure_text(base32.encode(nodeid))
|
||||
|
||||
def shortnodeid_b2a(nodeid):
|
||||
"""
|
||||
Short version of nodeid_b2a() output, Unicode string.
|
||||
"""
|
||||
return nodeid_b2a(nodeid)[:8]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user