ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-18 10:46:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #847 from tahoe-lafs/3459.test-checker-python-3

Browse Source 
Port test_checker.py to Python 3

Fixes ticket:3459
This commit is contained in:
Itamar Turner-Trauring 2020-10-16 10:31:25 -04:00 committed by GitHub
commit c2fe5a65a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 178 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3459.minor Normal file
View File

7
src/allmydata/check_results.py
View File

@ -1,3 +1,4 @@
from past.builtins import unicode
from zope.interface import implementer
from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
@ -56,7 +57,11 @@ class CheckResults(object):
self._list_incompatible_shares = list_incompatible_shares
self._count_incompatible_shares = count_incompatible_shares
assert isinstance(summary, str) # should be a single string
# On Python 2, we can mix bytes and Unicode. On Python 3, we want
# unicode.
if isinstance(summary, bytes):
summary = unicode(summary, "utf-8")
assert isinstance(summary, unicode) # should be a single string
self._summary = summary
assert not isinstance(report, str) # should be list of strings
self._report = report

2
src/allmydata/immutable/checker.py
View File

@ -616,7 +616,7 @@ class Checker(log.PrefixingLogMixin):
d.addCallback(_got_ueb)
def _discard_result(r):
assert isinstance(r, str), r
assert isinstance(r, bytes), r
# to free up the RAM
return None

1
src/allmydata/immutable/filenode.py
View File

@ -152,7 +152,6 @@ class CiphertextFileNode(object):
for server in servers:
sm.add(shnum, server)
servers_responding.add(server)
servers_responding = sorted(servers_responding)
good_hosts = len(reduce(set.union, sm.values(), set()))
is_healthy = bool(len(sm) >= verifycap.total_shares)

2
src/allmydata/mutable/publish.py
View File

@ -702,8 +702,8 @@ class Publish(object):
self.log("Pushing segment %d of %d" % (segnum + 1, self.num_segments))
# XXX: Why does this return a list?
data = self.data.read(segsize)
# XXX: This is dumb. Why return a list?
data = b"".join(data)
assert len(data) == segsize, len(data)

88
src/allmydata/test/test_checker.py
View File

@ -1,3 +1,16 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import json
import os.path, shutil
@ -7,7 +20,14 @@ from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.internet import defer
from nevow.inevow import IRequest
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from zope.interface import implementer
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
@ -102,7 +122,7 @@ class FakeCheckResults(object):
def get_corrupt_shares(self):
# returns a list of (IServer, storage_index, sharenum)
return [(FakeServer(), "<fake-si>", 0)]
return [(FakeServer(), b"<fake-si>", 0)]
@implementer(ICheckAndRepairResults)
@ -141,18 +161,18 @@ class WebResultsRendering(unittest.TestCase):
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
("v0-ffffffff-long", "\xff"*20, "peer-f"),
("v0-11111111-long", "\x11"*20, "peer-11")]
servers = [(b"v0-00000000-long", b"\x00"*20, "peer-0"),
(b"v0-ffffffff-long", b"\xff"*20, "peer-f"),
(b"v0-11111111-long", b"\x11"*20, "peer-11")]
for (key_s, binary_tubid, nickname) in servers:
server_id = key_s
tubid_b32 = base32.b2a(binary_tubid)
furl = "pb://%s@nowhere/fake" % tubid_b32
furl = b"pb://%s@nowhere/fake" % tubid_b32
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"nickname": str(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
@ -174,11 +194,11 @@ class WebResultsRendering(unittest.TestCase):
lcr = web_check_results.LiteralCheckResultsRendererElement()
html = self.render_element(lcr)
self.failUnlessIn("Literal files are always healthy", html)
self.failUnlessIn(b"Literal files are always healthy", html)
html = self.render_element(lcr, args={"return_to": ["FOOURL"]})
self.failUnlessIn("Literal files are always healthy", html)
self.failUnlessIn('<a href="FOOURL">Return to file.</a>', html)
self.failUnlessIn(b"Literal files are always healthy", html)
self.failUnlessIn(b'<a href="FOOURL">Return to file.</a>', html)
c = self.create_fake_client()
lcr = web_check_results.LiteralCheckResultsRenderer(c)
@ -192,11 +212,11 @@ class WebResultsRendering(unittest.TestCase):
def test_check(self):
c = self.create_fake_client()
sb = c.storage_broker
serverid_1 = "\x00"*20
serverid_f = "\xff"*20
serverid_1 = b"\x00"*20
serverid_f = b"\xff"*20
server_1 = sb.get_stub_server(serverid_1)
server_f = sb.get_stub_server(serverid_f)
u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
u = uri.CHKFileURI(b"\x00"*16, b"\x00"*32, 3, 10, 1234)
data = { "count_happiness": 8,
"count_shares_needed": 3,
"count_shares_expected": 9,
@ -260,7 +280,7 @@ class WebResultsRendering(unittest.TestCase):
self.failUnlessIn("Not Recoverable! : rather dead", s)
html = self.render_element(w, args={"return_to": ["FOOURL"]})
self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>',
self.failUnlessIn(b'<a href="FOOURL">Return to file/directory.</a>',
html)
w = web_check_results.CheckResultsRenderer(c, cr)
@ -301,9 +321,9 @@ class WebResultsRendering(unittest.TestCase):
def test_check_and_repair(self):
c = self.create_fake_client()
sb = c.storage_broker
serverid_1 = "\x00"*20
serverid_f = "\xff"*20
u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
serverid_1 = b"\x00"*20
serverid_f = b"\xff"*20
u = uri.CHKFileURI(b"\x00"*16, b"\x00"*32, 3, 10, 1234)
data = { "count_happiness": 5,
"count_shares_needed": 3,
@ -419,21 +439,21 @@ class WebResultsRendering(unittest.TestCase):
def test_deep_check_renderer(self):
status = check_results.DeepCheckResults("fake-root-si")
status = check_results.DeepCheckResults(b"fake-root-si")
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, False),
FakeCheckResults(b"<unhealthy/unrecoverable>", False, False),
(u"fake", u"unhealthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<healthy/recoverable>", True, True),
FakeCheckResults(b"<healthy/recoverable>", True, True),
(u"fake", u"healthy", u"recoverable")
)
status.add_check(
FakeCheckResults("<healthy/unrecoverable>", True, False),
FakeCheckResults(b"<healthy/unrecoverable>", True, False),
(u"fake", u"healthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, True),
FakeCheckResults(b"<unhealthy/unrecoverable>", False, True),
(u"fake", u"unhealthy", u"recoverable")
)
@ -512,18 +532,18 @@ class WebResultsRendering(unittest.TestCase):
)
def test_deep_check_and_repair_renderer(self):
status = check_results.DeepCheckAndRepairResults("")
status = check_results.DeepCheckAndRepairResults(b"")
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/success", True, True),
FakeCheckAndRepairResults(b"attempted/success", True, True),
(u"attempted", u"success")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/failure", True, False),
FakeCheckAndRepairResults(b"attempted/failure", True, False),
(u"attempted", u"failure")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("unattempted/failure", False, False),
FakeCheckAndRepairResults(b"unattempted/failure", False, False),
(u"unattempted", u"failure")
)
@ -662,7 +682,7 @@ class BalancingAct(GridTestMixin, unittest.TestCase):
"This little printing function is only meant for < 26 servers"
shares_chart = {}
names = dict(zip([ss.my_nodeid
for _,ss in self.g.servers_by_number.iteritems()],
for _,ss in self.g.servers_by_number.items()],
letters))
for shnum, serverid, _ in self.find_uri_shares(uri):
shares_chart.setdefault(shnum, []).append(names[serverid])
@ -676,8 +696,8 @@ class BalancingAct(GridTestMixin, unittest.TestCase):
c0.encoding_params['n'] = 4
c0.encoding_params['k'] = 3
DATA = "data" * 100
d = c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(Data(DATA, convergence=b""))
def _stash_immutable(ur):
self.imm = c0.create_node_from_uri(ur.get_uri())
self.uri = self.imm.get_uri()
@ -742,13 +762,13 @@ class AddLease(GridTestMixin, unittest.TestCase):
c0 = self.g.clients[0]
c0.encoding_params['happy'] = 1
self.uris = {}
DATA = "data" * 100
d = c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(Data(DATA, convergence=b""))
def _stash_immutable(ur):
self.imm = c0.create_node_from_uri(ur.get_uri())
d.addCallback(_stash_immutable)
d.addCallback(lambda ign:
c0.create_mutable_file(MutableData("contents")))
c0.create_mutable_file(MutableData(b"contents")))
def _stash_mutable(node):
self.mut = node
d.addCallback(_stash_mutable)
@ -834,8 +854,8 @@ class TooParallel(GridTestMixin, unittest.TestCase):
"max_segment_size": 5,
}
self.uris = {}
DATA = "data" * 100 # 400/5 = 80 blocks
return self.c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100 # 400/5 = 80 blocks
return self.c0.upload(Data(DATA, convergence=b""))
d.addCallback(_start)
def _do_check(ur):
n = self.c0.create_node_from_uri(ur.get_uri())

29
src/allmydata/test/test_util.py
View File

@ -14,16 +14,19 @@ if PY2:
import six
import os, time, sys
import yaml
import json
from twisted.trial import unittest
from allmydata.util import idlib, mathutil
from allmydata.util import fileutil
from allmydata.util import jsonbytes
from allmydata.util import pollmixin
from allmydata.util import yamlutil
from allmydata.util.fileutil import EncryptedTemporaryFile
from allmydata.test.common_util import ReallyEqualMixin
if six.PY3:
long = int
@ -470,3 +473,29 @@ class YAML(unittest.TestCase):
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
self.assertIsInstance(back[2], str)
class JSONBytes(unittest.TestCase):
"""Tests for BytesJSONEncoder."""
def test_encode_bytes(self):
"""BytesJSONEncoder can encode bytes."""
data = {
b"hello": [1, b"cd"],
}
expected = {
u"hello": [1, u"cd"],
}
# Bytes get passed through as if they were UTF-8 Unicode:
encoded = jsonbytes.dumps(data)
self.assertEqual(json.loads(encoded), expected)
self.assertEqual(jsonbytes.loads(encoded), expected)
def test_encode_unicode(self):
"""BytesJSONEncoder encodes Unicode string as usual."""
expected = {
u"hello": [1, u"cd"],
}
encoded = jsonbytes.dumps(expected)
self.assertEqual(json.loads(encoded), expected)

2
src/allmydata/util/_python3.py
View File

@ -76,6 +76,7 @@ PORTED_MODULES = [
"allmydata.util.hashutil",
"allmydata.util.humanreadable",
"allmydata.util.iputil",
"allmydata.util.jsonbytes",
"allmydata.util.log",
"allmydata.util.mathutil",
"allmydata.util.namespace",
@ -95,6 +96,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_abbreviate",
"allmydata.test.test_base32",
"allmydata.test.test_base62",
"allmydata.test.test_checker",
"allmydata.test.test_codec",
"allmydata.test.test_common_util",
"allmydata.test.test_configutil",

51
src/allmydata/util/jsonbytes.py Normal file
View File

@ -0,0 +1,51 @@
"""
A JSON encoder than can serialize bytes.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import json
class BytesJSONEncoder(json.JSONEncoder):
"""
A JSON encoder than can also encode bytes.
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
def default(self, o):
if isinstance(o, bytes):
return o.decode("utf-8")
return json.JSONEncoder.default(self, o)
def dumps(obj, *args, **kwargs):
"""Encode to JSON, supporting bytes as keys or values.
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
if isinstance(obj, dict):
new_obj = {}
for k, v in obj.items():
if isinstance(k, bytes):
k = k.decode("utf-8")
new_obj[k] = v
obj = new_obj
return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
# To make this module drop-in compatible with json module:
loads = json.loads
__all__ = ["dumps", "loads"]

7
src/allmydata/web/check_results.py
View File

@ -1,6 +1,6 @@
from future.builtins import str
import time
import json
from twisted.web import (
http,
@ -31,6 +31,7 @@ from allmydata.interfaces import (
from allmydata.util import (
base32,
dictutil,
jsonbytes as json, # Supporting dumping bytes
)
@ -200,7 +201,7 @@ class ResultsBase(object):
return tags.ul(r)
def _html(self, s):
if isinstance(s, (str, unicode)):
if isinstance(s, (bytes, str)):
return html.escape(s)
assert isinstance(s, (list, tuple))
return [html.escape(w) for w in s]
@ -522,7 +523,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin):
summary = cr.get_summary()
if summary:
summary_text = ": " + summary
summary_text += " [SI: %s]" % cr.get_storage_index_string()
summary_text += " [SI: %s]" % cr.get_storage_index_string().decode("ascii")
problems.append({
# Not sure self._join_pathstring(path) is the
# right thing to use here.

29
src/allmydata/web/common.py
View File

@ -1,3 +1,5 @@
from future.utils import PY2
from past.builtins import unicode
import time
import json
@ -9,9 +11,17 @@ from twisted.web import (
server,
template,
)
from twisted.web.iweb import IRequest as ITwistedRequest
from twisted.python import log
from nevow import appserver
from nevow.inevow import IRequest
if PY2:
from nevow.appserver import DefaultExceptionHandler
from nevow.inevow import IRequest as INevowRequest
else:
class DefaultExceptionHandler:
def __init__(self, *args, **kwargs):
raise NotImplementedError("Still not ported to Python 3")
INevowRequest = None
from allmydata import blacklist
from allmydata.interfaces import (
EmptyPathnameComponentError,
@ -118,7 +128,10 @@ def parse_offset_arg(offset):
def get_root(ctx_or_req):
req = IRequest(ctx_or_req)
if PY2:
req = INevowRequest(ctx_or_req)
else:
req = ITwistedRequest(ctx_or_req)
depth = len(req.prepath) + len(req.postpath)
link = "/".join([".."] * depth)
return link
@ -319,9 +332,9 @@ def humanize_failure(f):
return humanize_exception(f.value)
class MyExceptionHandler(appserver.DefaultExceptionHandler, object):
class MyExceptionHandler(DefaultExceptionHandler, object):
def simple(self, ctx, text, code=http.BAD_REQUEST):
req = IRequest(ctx)
req = INevowRequest(ctx)
req.setResponseCode(code)
#req.responseHeaders.setRawHeaders("content-encoding", [])
#req.responseHeaders.setRawHeaders("content-disposition", [])
@ -347,17 +360,17 @@ class MyExceptionHandler(appserver.DefaultExceptionHandler, object):
# twisted.web.server.Request.render() has support for transforming
# this into an appropriate 501 NOT_IMPLEMENTED or 405 NOT_ALLOWED
# return code, but nevow does not.
req = IRequest(ctx)
req = INevowRequest(ctx)
method = req.method
return self.simple(ctx,
"I don't know how to treat a %s request." % method,
http.NOT_IMPLEMENTED)
req = IRequest(ctx)
req = INevowRequest(ctx)
accept = req.getHeader("accept")
if not accept:
accept = "*/*"
if "*/*" in accept or "text/*" in accept or "text/html" in accept:
super = appserver.DefaultExceptionHandler
super = DefaultExceptionHandler
return super.renderHTTP_exception(self, ctx, f)
# use plain text
traceback = f.getTraceback()

14
src/allmydata/web/operations.py
View File

@ -1,6 +1,10 @@
from future.utils import PY2
import time
from nevow import url
if PY2:
from nevow import url
else:
# This module still needs porting to Python 3
url = None
from twisted.web.template import (
renderer,
tags as T,
@ -160,12 +164,12 @@ class ReloadMixin(object):
@renderer
def reload(self, req, tag):
if self.monitor.is_finished():
return ""
return b""
# url.gethere would break a proxy, so the correct thing to do is
# req.path[-1] + queryargs
ophandle = req.prepath[-1]
reload_target = ophandle + "?output=html"
cancel_target = ophandle + "?t=cancel"
reload_target = ophandle + b"?output=html"
cancel_target = ophandle + b"?t=cancel"
cancel_button = T.form(T.input(type="submit", value="Cancel"),
action=cancel_target,
method="POST",