ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-07 10:56:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #996 from tahoe-lafs/3628.port-more-tests-python-3

Browse Source 
Port more tests to Python 3

Fixes ticket:3628
This commit is contained in:
Itamar Turner-Trauring 2021-03-12 15:07:35 -05:00 committed by GitHub
commit 9bb30d2246
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
src/allmydata/test/test_import.py → newsfragments/3628.minor
View File

2
src/allmydata/test/cli/common.py
View File

@ -52,6 +52,8 @@ class CLITestMixin(ReallyEqualMixin):
# Python functions want native strings. So ignore the requirements
# for passing arguments to another process and make sure this argument
# is a native string.
verb = ensure_str(verb)
args = [ensure_str(arg) for arg in args]
client_dir = ensure_str(self.get_clientdir(i=client_num))
nodeargs = [ b"--node-directory", client_dir ]
return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)

2
src/allmydata/test/common.py
View File

@ -1059,7 +1059,7 @@ def _corrupt_mutable_share_data(data, debug=False):
assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
data_offset = MutableShareFile.DATA_OFFSET
sharetype = data[data_offset:data_offset+1]
assert sharetype == "\x00", "non-SDMF mutable shares not supported"
assert sharetype == b"\x00", "non-SDMF mutable shares not supported"
(version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
ig_datalen, offsets) = unpack_header(data[data_offset:])
assert version == 0, "this function only handles v0 SDMF files"

142
src/allmydata/test/test_deepcheck.py
View File

@ -1,4 +1,10 @@
import os, json, urllib
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
# Python 2 compatibility
# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
@ -11,7 +17,13 @@ import os, json, urllib
# (Pdb) pp data
# '334:12:b\'mutable-good\',90:URI:SSK-RO:...
from past.builtins import unicode as str
from future.utils import native_str
from future.utils import PY3, PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
import os, json
from urllib.parse import quote as url_quote
from twisted.trial import unittest
from twisted.internet import defer
@ -29,31 +41,37 @@ from allmydata.uri import LiteralFileURI
from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
ShouldFailMixin
from .common_util import StallMixin, run_cli
from .common_util import StallMixin, run_cli_unicode
from .common_web import do_http
from allmydata.test.no_network import GridTestMixin
from .cli.common import CLITestMixin
def run_cli(verb, *argv):
"""Match usage in existing tests by accept *args."""
return run_cli_unicode(verb, list(argv))
class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_good(self):
self.basedir = "deepcheck/MutableChecker/good"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _created(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
d.addCallback(_created)
# now make sure the webapi verifier sees no problems
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_results(out):
self.failUnless("<span>Healthy : Healthy</span>" in out, out)
self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
self.failIf("Not Healthy!" in out, out)
self.failIf("Unhealthy" in out, out)
self.failIf("Corrupt Shares" in out, out)
self.failUnless(b"<span>Healthy : Healthy</span>" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq1-" in out, out)
self.failIf(b"Not Healthy!" in out, out)
self.failIf(b"Unhealthy" in out, out)
self.failIf(b"Corrupt Shares" in out, out)
d.addCallback(_got_results)
d.addErrback(self.explain_web_error)
return d
@ -61,12 +79,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_corrupt(self):
self.basedir = "deepcheck/MutableChecker/corrupt"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_corrupt(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
self.corrupt_shares_numbered(node.get_uri(), [0],
_corrupt_mutable_share_data)
d.addCallback(_stash_and_corrupt)
@ -74,9 +92,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_results(out):
self.failUnless("Not Healthy!" in out, out)
self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failUnless("Corrupt Shares:" in out, out)
self.failUnless(b"Not Healthy!" in out, out)
self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failUnless(b"Corrupt Shares:" in out, out)
d.addCallback(_got_results)
# now make sure the webapi repairer can fix it
@ -84,13 +102,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
self.GET(self.fileurl+"?t=check&verify=true&repair=true",
method="POST"))
def _got_repair_results(out):
self.failUnless("<div>Repair successful</div>" in out, out)
self.failUnless(b"<div>Repair successful</div>" in out, out)
d.addCallback(_got_repair_results)
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_postrepair_results(out):
self.failIf("Not Healthy!" in out, out)
self.failUnless("Recoverable Versions: 10*seq" in out, out)
self.failIf(b"Not Healthy!" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq" in out, out)
d.addCallback(_got_postrepair_results)
d.addErrback(self.explain_web_error)
@ -99,21 +117,21 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_delete_share(self):
self.basedir = "deepcheck/MutableChecker/delete_share"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_delete(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
self.delete_shares_numbered(node.get_uri(), [0])
d.addCallback(_stash_and_delete)
# now make sure the webapi checker notices it
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
method="POST"))
def _got_results(out):
self.failUnless("Not Healthy!" in out, out)
self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failIf("Corrupt Shares" in out, out)
self.failUnless(b"Not Healthy!" in out, out)
self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failIf(b"Corrupt Shares" in out, out)
d.addCallback(_got_results)
# now make sure the webapi repairer can fix it
@ -121,13 +139,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
self.GET(self.fileurl+"?t=check&verify=false&repair=true",
method="POST"))
def _got_repair_results(out):
self.failUnless("Repair successful" in out)
self.failUnless(b"Repair successful" in out)
d.addCallback(_got_repair_results)
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
method="POST"))
def _got_postrepair_results(out):
self.failIf("Not Healthy!" in out, out)
self.failUnless("Recoverable Versions: 10*seq" in out)
self.failIf(b"Not Healthy!" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq" in out)
d.addCallback(_got_postrepair_results)
d.addErrback(self.explain_web_error)
@ -152,7 +170,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
return data
def parse_streamed_json(self, s):
for unit in s.split("\n"):
for unit in s.split(b"\n"):
if not unit:
# stream should end with a newline, so split returns ""
continue
@ -165,14 +183,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks
def web(self, n, method="GET", **kwargs):
# returns (data, url)
url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()]))
data = yield do_http(method, url, browser_like_redirects=True)
returnValue((data,url))
@inlineCallbacks
def wait_for_operation(self, ophandle):
url = self.client_baseurls[0] + "operations/" + ophandle
url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status&output=JSON"
while True:
body = yield do_http("get", url)
@ -184,7 +202,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks
def get_operation_results(self, ophandle, output=None):
url = self.client_baseurls[0] + "operations/" + ophandle
url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status"
if output:
url += "&output=" + output
@ -220,36 +238,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
self.root_uri = n.get_uri()
d.addCallback(_created_root)
d.addCallback(lambda ign:
c0.create_mutable_file(MutableData("mutable file contents")))
c0.create_mutable_file(MutableData(b"mutable file contents")))
d.addCallback(lambda n: self.root.set_node(u"mutable", n))
def _created_mutable(n):
self.mutable = n
self.mutable_uri = n.get_uri()
d.addCallback(_created_mutable)
large = upload.Data("Lots of data\n" * 1000, None)
large = upload.Data(b"Lots of data\n" * 1000, None)
d.addCallback(lambda ign: self.root.add_file(u"large", large))
def _created_large(n):
self.large = n
self.large_uri = n.get_uri()
d.addCallback(_created_large)
small = upload.Data("Small enough for a LIT", None)
small = upload.Data(b"Small enough for a LIT", None)
d.addCallback(lambda ign: self.root.add_file(u"small", small))
def _created_small(n):
self.small = n
self.small_uri = n.get_uri()
d.addCallback(_created_small)
small2 = upload.Data("Small enough for a LIT too", None)
small2 = upload.Data(b"Small enough for a LIT too", None)
d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
def _created_small2(n):
self.small2 = n
self.small2_uri = n.get_uri()
d.addCallback(_created_small2)
empty_litdir_uri = "URI:DIR2-LIT:"
tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
empty_litdir_uri = b"URI:DIR2-LIT:"
tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
def _created_empty_lit_dir(n):
@ -292,7 +310,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
sorted(self.g.get_all_serverids()),
where)
all_serverids = set()
for (shareid, servers) in cr.get_sharemap().items():
for (shareid, servers) in list(cr.get_sharemap().items()):
all_serverids.update([s.get_serverid() for s in servers])
self.failUnlessEqual(sorted(all_serverids),
sorted(self.g.get_all_serverids()),
@ -397,14 +415,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
mutable = [f for f in files
if f["cap"] is not None
and f["cap"].startswith("URI:SSK:")][0]
self.failUnlessEqual(mutable["cap"], self.mutable_uri)
self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri)
self.failIfEqual(mutable["cap"], mutable["verifycap"])
self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
# for immutable file, verifycap==repaircap!=filecap
large = [f for f in files
if f["cap"] is not None
and f["cap"].startswith("URI:CHK:")][0]
self.failUnlessEqual(large["cap"], self.large_uri)
self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri)
self.failIfEqual(large["cap"], large["verifycap"])
self.failUnlessEqual(large["verifycap"], large["repaircap"])
self.check_stats_good(stats)
@ -524,7 +542,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnless("summary" in data, (where, data))
self.failUnlessEqual(data["summary"].lower(), "healthy",
"%s: '%s'" % (where, data["summary"]))
@ -550,7 +568,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
where)
self.failUnless("sharemap" in r, where)
all_serverids = set()
for (shareid, serverids_s) in r["sharemap"].items():
for (shareid, serverids_s) in list(r["sharemap"].items()):
all_serverids.update(serverids_s)
self.failUnlessEqual(sorted(all_serverids),
sorted([idlib.nodeid_b2a(sid)
@ -562,7 +580,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["repair-attempted"], False, where)
self.json_check_is_healthy(data["pre-repair-results"],
n, where, incomplete)
@ -571,7 +589,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy"], 3, where)
self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
@ -582,7 +600,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
@ -728,6 +746,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def do_test_cli_good(self, ignored):
d = defer.succeed(None)
if PY3: # TODO fixme once Python 3 CLI porting is done
return d
d.addCallback(lambda ign: self.do_cli_manifest_stream1())
d.addCallback(lambda ign: self.do_cli_manifest_stream2())
d.addCallback(lambda ign: self.do_cli_manifest_stream3())
@ -738,7 +758,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def _check_manifest_storage_index(self, out):
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
@ -749,7 +769,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 8)
caps = {}
for l in lines:
@ -794,7 +814,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_verify_cap().to_string() in lines)
self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
@ -807,7 +827,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_repair_cap().to_string() in lines)
self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
@ -819,7 +839,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d = self.do_cli("stats", self.root_uri)
def _check3(args):
(rc, out, err) = args
lines = [l.strip() for l in out.split("\n") if l]
lines = [l.strip() for l in out.split(b"\n") if l]
self.failUnless("count-immutable-files: 1" in lines)
self.failUnless("count-mutable-files: 1" in lines)
self.failUnless("count-literal-files: 3" in lines)
@ -905,17 +925,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
d.addCallback(self.create_mangled, "large-unrecoverable")
d.addCallback(lambda ignored: c0.create_dirnode())
d.addCallback(self._stash_node, "broken")
large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None)
d.addCallback(lambda ignored:
self.nodes["broken"].add_file(u"large1", large1))
d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-good"))
large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
d.addCallback(self._stash_node, "subdir-unrecoverable")
large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
d.addCallback(lambda ignored:
self._delete_most_shares(self.nodes["broken"]))
@ -928,14 +948,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
def create_mangled(self, ignored, name):
nodetype, mangletype = name.split("-", 1)
if nodetype == "mutable":
mutable_uploadable = MutableData("mutable file contents")
mutable_uploadable = MutableData(b"mutable file contents")
d = self.g.clients[0].create_mutable_file(mutable_uploadable)
d.addCallback(lambda n: self.root.set_node(str(name), n))
d.addCallback(lambda n: self.root.set_node(str(name), n)) # TODO drop str() once strings are unicode
elif nodetype == "large":
large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None)
d = self.root.add_file(str(name), large)
elif nodetype == "small":
small = upload.Data("Small enough for a LIT", None)
small = upload.Data(b"Small enough for a LIT", None)
d = self.root.add_file(str(name), small)
d.addCallback(self._stash_node, name)
@ -959,10 +979,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
def _corrupt_some_shares(self, node):
for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
if shnum in (0,1):
yield run_cli("debug", "corrupt-share", native_str(sharefile))
yield run_cli("debug", "corrupt-share", sharefile)
def _delete_most_shares(self, node):
self.delete_shares_numbered(node.get_uri(), range(1,10))
self.delete_shares_numbered(node.get_uri(), list(range(1,10)))
def check_is_healthy(self, cr, where):
@ -1199,11 +1219,11 @@ class Large(DeepCheckBase, unittest.TestCase):
self.subdir_node = subdir_node
kids = {}
for i in range(1, COUNT):
litcap = LiteralFileURI("%03d-data" % i).to_string()
litcap = LiteralFileURI(b"%03d-data" % i).to_string()
kids[u"%03d-small" % i] = (litcap, litcap)
return subdir_node.set_children(kids)
d.addCallback(_add_children)
up = upload.Data("large enough for CHK" * 100, "")
up = upload.Data(b"large enough for CHK" * 100, b"")
d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
def _start_deepcheck(ignored):

6
src/allmydata/util/_python3.py
View File

@ -161,6 +161,11 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_connection_status",
"allmydata.test.test_crawler",
"allmydata.test.test_crypto",
# Only partially ported, CLI-using test code is disabled for now until CLI
# is ported.
"allmydata.test.test_deepcheck",
"allmydata.test.test_deferredutil",
"allmydata.test.test_dictutil",
"allmydata.test.test_dirnode",
@ -214,4 +219,5 @@ PORTED_TEST_MODULES = [
"allmydata.test.web.test_util",
"allmydata.test.web.test_web",
"allmydata.test.web.test_webish",
"allmydata.test.test_windows",
]