ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-08 11:24:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 3428.dont-return-deferred-to-twistedweb.2

Browse Source
This commit is contained in:
Jean-Paul Calderone 2020-10-16 10:37:54 -04:00
commit 32051f93b9
14 changed files with 244 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3459.minor Normal file
View File

0
newsfragments/3460.minor Normal file
View File

7
src/allmydata/check_results.py
View File

@ -1,3 +1,4 @@
from past.builtins import unicode
from zope.interface import implementer
from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
@ -56,7 +57,11 @@ class CheckResults(object):
self._list_incompatible_shares = list_incompatible_shares
self._count_incompatible_shares = count_incompatible_shares
assert isinstance(summary, str) # should be a single string
# On Python 2, we can mix bytes and Unicode. On Python 3, we want
# unicode.
if isinstance(summary, bytes):
summary = unicode(summary, "utf-8")
assert isinstance(summary, unicode) # should be a single string
self._summary = summary
assert not isinstance(report, str) # should be list of strings
self._report = report

2
src/allmydata/immutable/checker.py
View File

@ -616,7 +616,7 @@ class Checker(log.PrefixingLogMixin):
d.addCallback(_got_ueb)
def _discard_result(r):
assert isinstance(r, str), r
assert isinstance(r, bytes), r
# to free up the RAM
return None

1
src/allmydata/immutable/filenode.py
View File

@ -152,7 +152,6 @@ class CiphertextFileNode(object):
for server in servers:
sm.add(shnum, server)
servers_responding.add(server)
servers_responding = sorted(servers_responding)
good_hosts = len(reduce(set.union, sm.values(), set()))
is_healthy = bool(len(sm) >= verifycap.total_shares)

2
src/allmydata/mutable/publish.py
View File

@ -702,8 +702,8 @@ class Publish(object):
self.log("Pushing segment %d of %d" % (segnum + 1, self.num_segments))
# XXX: Why does this return a list?
data = self.data.read(segsize)
# XXX: This is dumb. Why return a list?
data = b"".join(data)
assert len(data) == segsize, len(data)

92
src/allmydata/storage_client.py
View File

@ -189,7 +189,10 @@ class StorageFarmBroker(service.MultiService):
# this sorted order).
for (server_id, server) in sorted(servers.items()):
try:
storage_server = self._make_storage_server(server_id, server)
storage_server = self._make_storage_server(
server_id.encode("utf-8"),
server,
)
except Exception:
# TODO: The _make_storage_server failure is logged but maybe
# we should write a traceback here. Notably, tests don't
@ -232,8 +235,19 @@ class StorageFarmBroker(service.MultiService):
include_result=False,
)
def _make_storage_server(self, server_id, server):
assert isinstance(server_id, unicode) # from YAML
server_id = server_id.encode("ascii")
"""
Create a new ``IServer`` for the given storage server announcement.
:param bytes server_id: The unique identifier for the server.
:param dict server: The server announcement. See ``Static Server
Definitions`` in the configuration documentation for details about
the structure and contents.
:return IServer: The object-y representation of the server described
by the given announcement.
"""
assert isinstance(server_id, bytes)
handler_overrides = server.get("connections", {})
s = NativeStorageServer(
server_id,
@ -260,7 +274,7 @@ class StorageFarmBroker(service.MultiService):
# these two are used in unit tests
def test_add_rref(self, serverid, rref, ann):
s = self._make_storage_server(
serverid.decode("ascii"),
serverid,
{"ann": ann.copy()},
)
s._rref = rref
@ -292,28 +306,71 @@ class StorageFarmBroker(service.MultiService):
remaining.append( (threshold, d) )
self._threshold_listeners = remaining
def _got_announcement(self, key_s, ann):
precondition(isinstance(key_s, str), key_s)
precondition(key_s.startswith("v0-"), key_s)
precondition(ann["service-name"] == "storage", ann["service-name"])
server_id = key_s
def _should_ignore_announcement(self, server_id, ann):
"""
Determine whether a new storage announcement should be discarded or used
to update our collection of storage servers.
:param bytes server_id: The unique identifier for the storage server
which made the announcement.
:param dict ann: The announcement.
:return bool: ``True`` if the announcement should be ignored,
``False`` if it should be used to update our local storage server
state.
"""
# Let local static configuration always override any announcement for
# a particular server.
if server_id in self._static_server_ids:
log.msg(format="ignoring announcement for static server '%(id)s'",
id=server_id,
facility="tahoe.storage_broker", umid="AlxzqA",
level=log.UNUSUAL)
return True
try:
old = self.servers[server_id]
except KeyError:
# We don't know anything about this server. Let's use the
# announcement to change that.
return False
else:
# Determine if this announcement is at all difference from the
# announcement we already have for the server. If it is the same,
# we don't need to change anything.
return old.get_announcement() == ann
def _got_announcement(self, key_s, ann):
"""
This callback is given to the introducer and called any time an
announcement is received which has a valid signature and does not have
a sequence number less than or equal to a previous sequence number
seen for that server by that introducer.
Note sequence numbers are not considered between different introducers
so if we use more than one introducer it is possible for them to
deliver us stale announcements in some cases.
"""
precondition(isinstance(key_s, str), key_s)
precondition(key_s.startswith("v0-"), key_s)
precondition(ann["service-name"] == "storage", ann["service-name"])
server_id = key_s
if self._should_ignore_announcement(server_id, ann):
return
s = self._make_storage_server(
server_id.decode("utf-8"),
server_id,
{u"ann": ann},
)
server_id = s.get_serverid()
old = self.servers.get(server_id)
if old:
if old.get_announcement() == ann:
return # duplicate
# replacement
del self.servers[server_id]
try:
old = self.servers.pop(server_id)
except KeyError:
pass
else:
# It's a replacement, get rid of the old one.
old.stop_connecting()
old.disownServiceParent()
# NOTE: this disownServiceParent() returns a Deferred that
@ -328,6 +385,7 @@ class StorageFarmBroker(service.MultiService):
# until they have fired (but hopefully don't keep reference
# cycles around when they fire earlier than that, which will
# almost always be the case for normal runtime).
# now we forget about them and start using the new one
s.setServiceParent(self)
self.servers[server_id] = s

88
src/allmydata/test/test_checker.py
View File

@ -1,3 +1,16 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import json
import os.path, shutil
@ -7,7 +20,14 @@ from bs4 import BeautifulSoup
from twisted.trial import unittest
from twisted.internet import defer
from nevow.inevow import IRequest
# We need to use `nevow.inevow.IRequest` for now for compatibility
# with the code in web/common.py. Once nevow bits are gone from
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
if PY2:
from nevow.inevow import IRequest
else:
from twisted.web.iweb import IRequest
from zope.interface import implementer
from twisted.web.server import Request
from twisted.web.test.requesthelper import DummyChannel
@ -105,7 +125,7 @@ class FakeCheckResults(object):
def get_corrupt_shares(self):
# returns a list of (IServer, storage_index, sharenum)
return [(FakeServer(), "<fake-si>", 0)]
return [(FakeServer(), b"<fake-si>", 0)]
@implementer(ICheckAndRepairResults)
@ -144,18 +164,18 @@ class WebResultsRendering(unittest.TestCase):
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
# s.get_name() (the "short description") will be "v0-00000000".
# s.get_longname() will include the -long suffix.
servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
("v0-ffffffff-long", "\xff"*20, "peer-f"),
("v0-11111111-long", "\x11"*20, "peer-11")]
servers = [(b"v0-00000000-long", b"\x00"*20, "peer-0"),
(b"v0-ffffffff-long", b"\xff"*20, "peer-f"),
(b"v0-11111111-long", b"\x11"*20, "peer-11")]
for (key_s, binary_tubid, nickname) in servers:
server_id = key_s
tubid_b32 = base32.b2a(binary_tubid)
furl = "pb://%s@nowhere/fake" % tubid_b32
furl = b"pb://%s@nowhere/fake" % tubid_b32
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,
"permutation-seed-base32": "",
"nickname": unicode(nickname),
"nickname": str(nickname),
"app-versions": {}, # need #466 and v2 introducer
"my-version": "ver",
"oldest-supported": "oldest",
@ -177,11 +197,11 @@ class WebResultsRendering(unittest.TestCase):
lcr = web_check_results.LiteralCheckResultsRendererElement()
html = self.render_element(lcr)
self.failUnlessIn("Literal files are always healthy", html)
self.failUnlessIn(b"Literal files are always healthy", html)
html = self.render_element(lcr, args={"return_to": ["FOOURL"]})
self.failUnlessIn("Literal files are always healthy", html)
self.failUnlessIn('<a href="FOOURL">Return to file.</a>', html)
self.failUnlessIn(b"Literal files are always healthy", html)
self.failUnlessIn(b'<a href="FOOURL">Return to file.</a>', html)
c = self.create_fake_client()
lcr = web_check_results.LiteralCheckResultsRenderer(c)
@ -195,11 +215,11 @@ class WebResultsRendering(unittest.TestCase):
def test_check(self):
c = self.create_fake_client()
sb = c.storage_broker
serverid_1 = "\x00"*20
serverid_f = "\xff"*20
serverid_1 = b"\x00"*20
serverid_f = b"\xff"*20
server_1 = sb.get_stub_server(serverid_1)
server_f = sb.get_stub_server(serverid_f)
u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
u = uri.CHKFileURI(b"\x00"*16, b"\x00"*32, 3, 10, 1234)
data = { "count_happiness": 8,
"count_shares_needed": 3,
"count_shares_expected": 9,
@ -263,7 +283,7 @@ class WebResultsRendering(unittest.TestCase):
self.failUnlessIn("Not Recoverable! : rather dead", s)
html = self.render_element(w, args={"return_to": ["FOOURL"]})
self.failUnlessIn('<a href="FOOURL">Return to file/directory.</a>',
self.failUnlessIn(b'<a href="FOOURL">Return to file/directory.</a>',
html)
w = web_check_results.CheckResultsRenderer(c, cr)
@ -304,9 +324,9 @@ class WebResultsRendering(unittest.TestCase):
def test_check_and_repair(self):
c = self.create_fake_client()
sb = c.storage_broker
serverid_1 = "\x00"*20
serverid_f = "\xff"*20
u = uri.CHKFileURI("\x00"*16, "\x00"*32, 3, 10, 1234)
serverid_1 = b"\x00"*20
serverid_f = b"\xff"*20
u = uri.CHKFileURI(b"\x00"*16, b"\x00"*32, 3, 10, 1234)
data = { "count_happiness": 5,
"count_shares_needed": 3,
@ -422,21 +442,21 @@ class WebResultsRendering(unittest.TestCase):
def test_deep_check_renderer(self):
status = check_results.DeepCheckResults("fake-root-si")
status = check_results.DeepCheckResults(b"fake-root-si")
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, False),
FakeCheckResults(b"<unhealthy/unrecoverable>", False, False),
(u"fake", u"unhealthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<healthy/recoverable>", True, True),
FakeCheckResults(b"<healthy/recoverable>", True, True),
(u"fake", u"healthy", u"recoverable")
)
status.add_check(
FakeCheckResults("<healthy/unrecoverable>", True, False),
FakeCheckResults(b"<healthy/unrecoverable>", True, False),
(u"fake", u"healthy", u"unrecoverable")
)
status.add_check(
FakeCheckResults("<unhealthy/unrecoverable>", False, True),
FakeCheckResults(b"<unhealthy/unrecoverable>", False, True),
(u"fake", u"unhealthy", u"recoverable")
)
@ -515,18 +535,18 @@ class WebResultsRendering(unittest.TestCase):
)
def test_deep_check_and_repair_renderer(self):
status = check_results.DeepCheckAndRepairResults("")
status = check_results.DeepCheckAndRepairResults(b"")
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/success", True, True),
FakeCheckAndRepairResults(b"attempted/success", True, True),
(u"attempted", u"success")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("attempted/failure", True, False),
FakeCheckAndRepairResults(b"attempted/failure", True, False),
(u"attempted", u"failure")
)
status.add_check_and_repair(
FakeCheckAndRepairResults("unattempted/failure", False, False),
FakeCheckAndRepairResults(b"unattempted/failure", False, False),
(u"unattempted", u"failure")
)
@ -665,7 +685,7 @@ class BalancingAct(GridTestMixin, unittest.TestCase):
"This little printing function is only meant for < 26 servers"
shares_chart = {}
names = dict(zip([ss.my_nodeid
for _,ss in self.g.servers_by_number.iteritems()],
for _,ss in self.g.servers_by_number.items()],
letters))
for shnum, serverid, _ in self.find_uri_shares(uri):
shares_chart.setdefault(shnum, []).append(names[serverid])
@ -679,8 +699,8 @@ class BalancingAct(GridTestMixin, unittest.TestCase):
c0.encoding_params['n'] = 4
c0.encoding_params['k'] = 3
DATA = "data" * 100
d = c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(Data(DATA, convergence=b""))
def _stash_immutable(ur):
self.imm = c0.create_node_from_uri(ur.get_uri())
self.uri = self.imm.get_uri()
@ -745,13 +765,13 @@ class AddLease(GridTestMixin, unittest.TestCase):
c0 = self.g.clients[0]
c0.encoding_params['happy'] = 1
self.uris = {}
DATA = "data" * 100
d = c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(Data(DATA, convergence=b""))
def _stash_immutable(ur):
self.imm = c0.create_node_from_uri(ur.get_uri())
d.addCallback(_stash_immutable)
d.addCallback(lambda ign:
c0.create_mutable_file(MutableData("contents")))
c0.create_mutable_file(MutableData(b"contents")))
def _stash_mutable(node):
self.mut = node
d.addCallback(_stash_mutable)
@ -837,8 +857,8 @@ class TooParallel(GridTestMixin, unittest.TestCase):
"max_segment_size": 5,
}
self.uris = {}
DATA = "data" * 100 # 400/5 = 80 blocks
return self.c0.upload(Data(DATA, convergence=""))
DATA = b"data" * 100 # 400/5 = 80 blocks
return self.c0.upload(Data(DATA, convergence=b""))
d.addCallback(_start)
def _do_check(ur):
n = self.c0.create_node_from_uri(ur.get_uri())

29
src/allmydata/test/test_util.py
View File

@ -14,16 +14,19 @@ if PY2:
import six
import os, time, sys
import yaml
import json
from twisted.trial import unittest
from allmydata.util import idlib, mathutil
from allmydata.util import fileutil
from allmydata.util import jsonbytes
from allmydata.util import pollmixin
from allmydata.util import yamlutil
from allmydata.util.fileutil import EncryptedTemporaryFile
from allmydata.test.common_util import ReallyEqualMixin
if six.PY3:
long = int
@ -470,3 +473,29 @@ class YAML(unittest.TestCase):
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
self.assertIsInstance(back[2], str)
class JSONBytes(unittest.TestCase):
"""Tests for BytesJSONEncoder."""
def test_encode_bytes(self):
"""BytesJSONEncoder can encode bytes."""
data = {
b"hello": [1, b"cd"],
}
expected = {
u"hello": [1, u"cd"],
}
# Bytes get passed through as if they were UTF-8 Unicode:
encoded = jsonbytes.dumps(data)
self.assertEqual(json.loads(encoded), expected)
self.assertEqual(jsonbytes.loads(encoded), expected)
def test_encode_unicode(self):
"""BytesJSONEncoder encodes Unicode string as usual."""
expected = {
u"hello": [1, u"cd"],
}
encoded = jsonbytes.dumps(expected)
self.assertEqual(json.loads(encoded), expected)

2
src/allmydata/util/_python3.py
View File

@ -76,6 +76,7 @@ PORTED_MODULES = [
"allmydata.util.hashutil",
"allmydata.util.humanreadable",
"allmydata.util.iputil",
"allmydata.util.jsonbytes",
"allmydata.util.log",
"allmydata.util.mathutil",
"allmydata.util.namespace",
@ -95,6 +96,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_abbreviate",
"allmydata.test.test_base32",
"allmydata.test.test_base62",
"allmydata.test.test_checker",
"allmydata.test.test_codec",
"allmydata.test.test_common_util",
"allmydata.test.test_configutil",

51
src/allmydata/util/jsonbytes.py Normal file
View File

@ -0,0 +1,51 @@
"""
A JSON encoder than can serialize bytes.
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import json
class BytesJSONEncoder(json.JSONEncoder):
"""
A JSON encoder than can also encode bytes.
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
def default(self, o):
if isinstance(o, bytes):
return o.decode("utf-8")
return json.JSONEncoder.default(self, o)
def dumps(obj, *args, **kwargs):
"""Encode to JSON, supporting bytes as keys or values.
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
if isinstance(obj, dict):
new_obj = {}
for k, v in obj.items():
if isinstance(k, bytes):
k = k.decode("utf-8")
new_obj[k] = v
obj = new_obj
return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)
# To make this module drop-in compatible with json module:
loads = json.loads
__all__ = ["dumps", "loads"]

7
src/allmydata/web/check_results.py
View File

@ -1,6 +1,6 @@
from future.builtins import str
import time
import json
from twisted.web import (
http,
@ -31,6 +31,7 @@ from allmydata.interfaces import (
from allmydata.util import (
base32,
dictutil,
jsonbytes as json, # Supporting dumping bytes
)
@ -200,7 +201,7 @@ class ResultsBase(object):
return tags.ul(r)
def _html(self, s):
if isinstance(s, (str, unicode)):
if isinstance(s, (bytes, str)):
return html.escape(s)
assert isinstance(s, (list, tuple))
return [html.escape(w) for w in s]
@ -522,7 +523,7 @@ class DeepCheckResultsRendererElement(Element, ResultsBase, ReloadMixin):
summary = cr.get_summary()
if summary:
summary_text = ": " + summary
summary_text += " [SI: %s]" % cr.get_storage_index_string()
summary_text += " [SI: %s]" % cr.get_storage_index_string().decode("ascii")
problems.append({
# Not sure self._join_pathstring(path) is the
# right thing to use here.

20
src/allmydata/web/common.py
View File

@ -1,3 +1,5 @@
from future.utils import PY2
from past.builtins import unicode
import time
import json
@ -51,6 +53,17 @@ from twisted.web.resource import (
from nevow import appserver
from nevow.inevow import IRequest
from twisted.web.iweb import IRequest as ITwistedRequest
from twisted.python import log
if PY2:
from nevow.appserver import DefaultExceptionHandler
from nevow.inevow import IRequest as INevowRequest
else:
class DefaultExceptionHandler:
def __init__(self, *args, **kwargs):
raise NotImplementedError("Still not ported to Python 3")
INevowRequest = None
from allmydata import blacklist
from allmydata.interfaces import (
EmptyPathnameComponentError,
@ -157,7 +170,10 @@ def parse_offset_arg(offset):
def get_root(ctx_or_req):
req = IRequest(ctx_or_req)
if PY2:
req = INevowRequest(ctx_or_req)
else:
req = ITwistedRequest(ctx_or_req)
depth = len(req.prepath) + len(req.postpath)
link = "/".join([".."] * depth)
return link
@ -358,7 +374,7 @@ def humanize_failure(f):
return humanize_exception(f.value)
class MyExceptionHandler(appserver.DefaultExceptionHandler, object):
class MyExceptionHandler(DefaultExceptionHandler, object):
def renderHTTP_exception(self, ctx, f):
req = IRequest(ctx)
req.write(_renderHTTP_exception(req, f))

6
src/allmydata/web/operations.py
View File

@ -161,12 +161,12 @@ class ReloadMixin(object):
@renderer
def reload(self, req, tag):
if self.monitor.is_finished():
return ""
return b""
# url.gethere would break a proxy, so the correct thing to do is
# req.path[-1] + queryargs
ophandle = req.prepath[-1]
reload_target = ophandle + "?output=html"
cancel_target = ophandle + "?t=cancel"
reload_target = ophandle + b"?output=html"
cancel_target = ophandle + b"?t=cancel"
cancel_button = T.form(T.input(type="submit", value="Cancel"),
action=cancel_target,
method="POST",