ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-07 20:00:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

A couple tests passing on Python 3.

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2021-03-09 11:10:48 -05:00
parent a4a8a22f8f
commit 6f264a60e3
1 changed files with 42 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/allmydata/test/test_deepcheck.py
View File

@ -1,5 +1,3 @@
import os, json, urllib
# Python 2 compatibility # Python 2 compatibility
# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing # Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
# a `future.types.newstr.newstr` as a *Python 3* byte string representation under # a `future.types.newstr.newstr` as a *Python 3* byte string representation under
@ -11,7 +9,10 @@ import os, json, urllib
# (Pdb) pp data # (Pdb) pp data
# '334:12:b\'mutable-good\',90:URI:SSK-RO:... # '334:12:b\'mutable-good\',90:URI:SSK-RO:...
from past.builtins import unicode as str from past.builtins import unicode as str
from future.utils import native_str from future.utils import native_str, PY3
import os, json
from urllib.parse import quote as url_quote
from twisted.trial import unittest from twisted.trial import unittest
from twisted.internet import defer from twisted.internet import defer
@ -38,12 +39,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_good(self): def test_good(self):
self.basedir = "deepcheck/MutableChecker/good" self.basedir = "deepcheck/MutableChecker/good"
self.set_up_grid() self.set_up_grid()
CONTENTS = "a little bit of data" CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS) CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _created(node): def _created(node):
self.node = node self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri()) self.fileurl = "uri/" + url_quote(node.get_uri())
d.addCallback(_created) d.addCallback(_created)
# now make sure the webapi verifier sees no problems # now make sure the webapi verifier sees no problems
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true", d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
@ -61,12 +62,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_corrupt(self): def test_corrupt(self):
self.basedir = "deepcheck/MutableChecker/corrupt" self.basedir = "deepcheck/MutableChecker/corrupt"
self.set_up_grid() self.set_up_grid()
CONTENTS = "a little bit of data" CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS) CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_corrupt(node): def _stash_and_corrupt(node):
self.node = node self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri()) self.fileurl = "uri/" + url_quote(node.get_uri())
self.corrupt_shares_numbered(node.get_uri(), [0], self.corrupt_shares_numbered(node.get_uri(), [0],
_corrupt_mutable_share_data) _corrupt_mutable_share_data)
d.addCallback(_stash_and_corrupt) d.addCallback(_stash_and_corrupt)
@ -99,12 +100,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_delete_share(self): def test_delete_share(self):
self.basedir = "deepcheck/MutableChecker/delete_share" self.basedir = "deepcheck/MutableChecker/delete_share"
self.set_up_grid() self.set_up_grid()
CONTENTS = "a little bit of data" CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS) CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable) d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_delete(node): def _stash_and_delete(node):
self.node = node self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri()) self.fileurl = "uri/" + url_quote(node.get_uri())
self.delete_shares_numbered(node.get_uri(), [0]) self.delete_shares_numbered(node.get_uri(), [0])
d.addCallback(_stash_and_delete) d.addCallback(_stash_and_delete)
# now make sure the webapi checker notices it # now make sure the webapi checker notices it
@ -152,7 +153,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
return data return data
def parse_streamed_json(self, s): def parse_streamed_json(self, s):
for unit in s.split("\n"): for unit in s.split(b"\n"):
if not unit: if not unit:
# stream should end with a newline, so split returns "" # stream should end with a newline, so split returns ""
continue continue
@ -165,14 +166,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks @inlineCallbacks
def web(self, n, method="GET", **kwargs): def web(self, n, method="GET", **kwargs):
# returns (data, url) # returns (data, url)
url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri()) url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()])) + "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()]))
data = yield do_http(method, url, browser_like_redirects=True) data = yield do_http(method, url, browser_like_redirects=True)
returnValue((data,url)) returnValue((data,url))
@inlineCallbacks @inlineCallbacks
def wait_for_operation(self, ophandle): def wait_for_operation(self, ophandle):
url = self.client_baseurls[0] + "operations/" + ophandle url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status&output=JSON" url += "?t=status&output=JSON"
while True: while True:
body = yield do_http("get", url) body = yield do_http("get", url)
@ -184,7 +185,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks @inlineCallbacks
def get_operation_results(self, ophandle, output=None): def get_operation_results(self, ophandle, output=None):
url = self.client_baseurls[0] + "operations/" + ophandle url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status" url += "?t=status"
if output: if output:
url += "&output=" + output url += "&output=" + output
@ -220,36 +221,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
self.root_uri = n.get_uri() self.root_uri = n.get_uri()
d.addCallback(_created_root) d.addCallback(_created_root)
d.addCallback(lambda ign: d.addCallback(lambda ign:
c0.create_mutable_file(MutableData("mutable file contents"))) c0.create_mutable_file(MutableData(b"mutable file contents")))
d.addCallback(lambda n: self.root.set_node(u"mutable", n)) d.addCallback(lambda n: self.root.set_node(u"mutable", n))
def _created_mutable(n): def _created_mutable(n):
self.mutable = n self.mutable = n
self.mutable_uri = n.get_uri() self.mutable_uri = n.get_uri()
d.addCallback(_created_mutable) d.addCallback(_created_mutable)
large = upload.Data("Lots of data\n" * 1000, None) large = upload.Data(b"Lots of data\n" * 1000, None)
d.addCallback(lambda ign: self.root.add_file(u"large", large)) d.addCallback(lambda ign: self.root.add_file(u"large", large))
def _created_large(n): def _created_large(n):
self.large = n self.large = n
self.large_uri = n.get_uri() self.large_uri = n.get_uri()
d.addCallback(_created_large) d.addCallback(_created_large)
small = upload.Data("Small enough for a LIT", None) small = upload.Data(b"Small enough for a LIT", None)
d.addCallback(lambda ign: self.root.add_file(u"small", small)) d.addCallback(lambda ign: self.root.add_file(u"small", small))
def _created_small(n): def _created_small(n):
self.small = n self.small = n
self.small_uri = n.get_uri() self.small_uri = n.get_uri()
d.addCallback(_created_small) d.addCallback(_created_small)
small2 = upload.Data("Small enough for a LIT too", None) small2 = upload.Data(b"Small enough for a LIT too", None)
d.addCallback(lambda ign: self.root.add_file(u"small2", small2)) d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
def _created_small2(n): def _created_small2(n):
self.small2 = n self.small2 = n
self.small2_uri = n.get_uri() self.small2_uri = n.get_uri()
d.addCallback(_created_small2) d.addCallback(_created_small2)
empty_litdir_uri = "URI:DIR2-LIT:" empty_litdir_uri = b"URI:DIR2-LIT:"
tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir")) d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
def _created_empty_lit_dir(n): def _created_empty_lit_dir(n):
@ -397,14 +398,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
mutable = [f for f in files mutable = [f for f in files
if f["cap"] is not None if f["cap"] is not None
and f["cap"].startswith("URI:SSK:")][0] and f["cap"].startswith("URI:SSK:")][0]
self.failUnlessEqual(mutable["cap"], self.mutable_uri) self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri)
self.failIfEqual(mutable["cap"], mutable["verifycap"]) self.failIfEqual(mutable["cap"], mutable["verifycap"])
self.failUnlessEqual(mutable["cap"], mutable["repaircap"]) self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
# for immutable file, verifycap==repaircap!=filecap # for immutable file, verifycap==repaircap!=filecap
large = [f for f in files large = [f for f in files
if f["cap"] is not None if f["cap"] is not None
and f["cap"].startswith("URI:CHK:")][0] and f["cap"].startswith("URI:CHK:")][0]
self.failUnlessEqual(large["cap"], self.large_uri) self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri)
self.failIfEqual(large["cap"], large["verifycap"]) self.failIfEqual(large["cap"], large["verifycap"])
self.failUnlessEqual(large["verifycap"], large["repaircap"]) self.failUnlessEqual(large["verifycap"], large["repaircap"])
self.check_stats_good(stats) self.check_stats_good(stats)
@ -524,7 +525,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_is_healthy(self, data, n, where, incomplete=False): def json_check_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"], self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where) str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnless("summary" in data, (where, data)) self.failUnless("summary" in data, (where, data))
self.failUnlessEqual(data["summary"].lower(), "healthy", self.failUnlessEqual(data["summary"].lower(), "healthy",
"%s: '%s'" % (where, data["summary"])) "%s: '%s'" % (where, data["summary"]))
@ -562,7 +563,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False): def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"], self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where) str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["repair-attempted"], False, where) self.failUnlessEqual(data["repair-attempted"], False, where)
self.json_check_is_healthy(data["pre-repair-results"], self.json_check_is_healthy(data["pre-repair-results"],
n, where, incomplete) n, where, incomplete)
@ -571,7 +572,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_is_healthy(self, data, n, where): def json_full_deepcheck_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"], self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where) str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy"], 3, where) self.failUnlessEqual(data["count-objects-healthy"], 3, where)
self.failUnlessEqual(data["count-objects-unhealthy"], 0, where) self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
@ -582,7 +583,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_and_repair_is_healthy(self, data, n, where): def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"], self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where) str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where) self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where) self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
@ -728,6 +729,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def do_test_cli_good(self, ignored): def do_test_cli_good(self, ignored):
d = defer.succeed(None) d = defer.succeed(None)
if PY3: # TODO fixme once Python 3 CLI porting is done
return d
d.addCallback(lambda ign: self.do_cli_manifest_stream1()) d.addCallback(lambda ign: self.do_cli_manifest_stream1())
d.addCallback(lambda ign: self.do_cli_manifest_stream2()) d.addCallback(lambda ign: self.do_cli_manifest_stream2())
d.addCallback(lambda ign: self.do_cli_manifest_stream3()) d.addCallback(lambda ign: self.do_cli_manifest_stream3())
@ -738,7 +741,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d return d
def _check_manifest_storage_index(self, out): def _check_manifest_storage_index(self, out):
lines = [l for l in out.split("\n") if l] lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3) self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines) self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines) self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
@ -749,7 +752,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args): def _check(args):
(rc, out, err) = args (rc, out, err) = args
self.failUnlessEqual(err, "") self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l] lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 8) self.failUnlessEqual(len(lines), 8)
caps = {} caps = {}
for l in lines: for l in lines:
@ -794,7 +797,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args): def _check(args):
(rc, out, err) = args (rc, out, err) = args
self.failUnlessEqual(err, "") self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l] lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3) self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_verify_cap().to_string() in lines) self.failUnless(self.root.get_verify_cap().to_string() in lines)
self.failUnless(self.mutable.get_verify_cap().to_string() in lines) self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
@ -807,7 +810,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args): def _check(args):
(rc, out, err) = args (rc, out, err) = args
self.failUnlessEqual(err, "") self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l] lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3) self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_repair_cap().to_string() in lines) self.failUnless(self.root.get_repair_cap().to_string() in lines)
self.failUnless(self.mutable.get_repair_cap().to_string() in lines) self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
@ -819,7 +822,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d = self.do_cli("stats", self.root_uri) d = self.do_cli("stats", self.root_uri)
def _check3(args): def _check3(args):
(rc, out, err) = args (rc, out, err) = args
lines = [l.strip() for l in out.split("\n") if l] lines = [l.strip() for l in out.split(b"\n") if l]
self.failUnless("count-immutable-files: 1" in lines) self.failUnless("count-immutable-files: 1" in lines)
self.failUnless("count-mutable-files: 1" in lines) self.failUnless("count-mutable-files: 1" in lines)
self.failUnless("count-literal-files: 3" in lines) self.failUnless("count-literal-files: 3" in lines)
@ -905,17 +908,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
d.addCallback(self.create_mangled, "large-unrecoverable") d.addCallback(self.create_mangled, "large-unrecoverable")
d.addCallback(lambda ignored: c0.create_dirnode()) d.addCallback(lambda ignored: c0.create_dirnode())
d.addCallback(self._stash_node, "broken") d.addCallback(self._stash_node, "broken")
large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None) large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None)
d.addCallback(lambda ignored: d.addCallback(lambda ignored:
self.nodes["broken"].add_file(u"large1", large1)) self.nodes["broken"].add_file(u"large1", large1))
d.addCallback(lambda ignored: d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-good")) self.nodes["broken"].create_subdirectory(u"subdir-good"))
large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None) large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2)) d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
d.addCallback(lambda ignored: d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable")) self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
d.addCallback(self._stash_node, "subdir-unrecoverable") d.addCallback(self._stash_node, "subdir-unrecoverable")
large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None) large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3)) d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
d.addCallback(lambda ignored: d.addCallback(lambda ignored:
self._delete_most_shares(self.nodes["broken"])) self._delete_most_shares(self.nodes["broken"]))
@ -928,14 +931,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
def create_mangled(self, ignored, name): def create_mangled(self, ignored, name):
nodetype, mangletype = name.split("-", 1) nodetype, mangletype = name.split("-", 1)
if nodetype == "mutable": if nodetype == "mutable":
mutable_uploadable = MutableData("mutable file contents") mutable_uploadable = MutableData(b"mutable file contents")
d = self.g.clients[0].create_mutable_file(mutable_uploadable) d = self.g.clients[0].create_mutable_file(mutable_uploadable)
d.addCallback(lambda n: self.root.set_node(str(name), n)) d.addCallback(lambda n: self.root.set_node(str(name), n)) # TODO drop str() once strings are unicode
elif nodetype == "large": elif nodetype == "large":
large = upload.Data("Lots of data\n" * 1000 + name + "\n", None) large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None)
d = self.root.add_file(str(name), large) d = self.root.add_file(str(name), large)
elif nodetype == "small": elif nodetype == "small":
small = upload.Data("Small enough for a LIT", None) small = upload.Data(b"Small enough for a LIT", None)
d = self.root.add_file(str(name), small) d = self.root.add_file(str(name), small)
d.addCallback(self._stash_node, name) d.addCallback(self._stash_node, name)
@ -1203,7 +1206,7 @@ class Large(DeepCheckBase, unittest.TestCase):
kids[u"%03d-small" % i] = (litcap, litcap) kids[u"%03d-small" % i] = (litcap, litcap)
return subdir_node.set_children(kids) return subdir_node.set_children(kids)
d.addCallback(_add_children) d.addCallback(_add_children)
up = upload.Data("large enough for CHK" * 100, "") up = upload.Data(b"large enough for CHK" * 100, "")
d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up)) d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
def _start_deepcheck(ignored): def _start_deepcheck(ignored):