ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-13 22:03:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #976 from tahoe-lafs/3596.test-web-python-3-even-more

Browse Source 
Port test_web.py to Python 3

Fixes ticket:3596
This commit is contained in:
Itamar Turner-Trauring 2021-02-10 14:21:22 -05:00 committed by GitHub
commit e5806301d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 495 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
misc/operations_helpers/provisioning/provisioning.py
View File

@ -46,7 +46,7 @@ class ProvisioningTool(rend.Page):
req = inevow.IRequest(ctx)
def getarg(name, astype=int):
if req.method != "POST":
if req.method != b"POST":
return None
if name in req.fields:
return astype(req.fields[name].value)

0
newsfragments/3596.minor Normal file
View File

18
src/allmydata/test/common.py
View File

@ -432,7 +432,7 @@ class FakeCHKFileNode(object): # type: ignore # incomplete implementation
return self.storage_index
def check(self, monitor, verify=False, add_lease=False):
s = StubServer("\x00"*20)
s = StubServer(b"\x00"*20)
r = CheckResults(self.my_uri, self.storage_index,
healthy=True, recoverable=True,
count_happiness=10,
@ -566,12 +566,12 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
self.file_types[self.storage_index] = version
initial_contents = self._get_initial_contents(contents)
data = initial_contents.read(initial_contents.get_size())
data = "".join(data)
data = b"".join(data)
self.all_contents[self.storage_index] = data
return defer.succeed(self)
def _get_initial_contents(self, contents):
if contents is None:
return MutableData("")
return MutableData(b"")
if IMutableUploadable.providedBy(contents):
return contents
@ -625,7 +625,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
def raise_error(self):
pass
def get_writekey(self):
return "\x00"*16
return b"\x00"*16
def get_size(self):
return len(self.all_contents[self.storage_index])
def get_current_size(self):
@ -644,7 +644,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
return self.file_types[self.storage_index]
def check(self, monitor, verify=False, add_lease=False):
s = StubServer("\x00"*20)
s = StubServer(b"\x00"*20)
r = CheckResults(self.my_uri, self.storage_index,
healthy=True, recoverable=True,
count_happiness=10,
@ -655,7 +655,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
count_recoverable_versions=1,
count_unrecoverable_versions=0,
servers_responding=[s],
sharemap={"seq1-abcd-sh0": [s]},
sharemap={b"seq1-abcd-sh0": [s]},
count_wrong_shares=0,
list_corrupt_shares=[],
count_corrupt_shares=0,
@ -709,7 +709,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
def overwrite(self, new_contents):
assert not self.is_readonly()
new_data = new_contents.read(new_contents.get_size())
new_data = "".join(new_data)
new_data = b"".join(new_data)
self.all_contents[self.storage_index] = new_data
return defer.succeed(None)
def modify(self, modifier):
@ -740,7 +740,7 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
def update(self, data, offset):
assert not self.is_readonly()
def modifier(old, servermap, first_time):
new = old[:offset] + "".join(data.read(data.get_size()))
new = old[:offset] + b"".join(data.read(data.get_size()))
new += old[len(new):]
return new
return self.modify(modifier)
@ -859,6 +859,8 @@ class WebErrorMixin(object):
body = yield response.content()
self.assertEquals(response.code, code)
if response_substring is not None:
if isinstance(response_substring, unicode):
response_substring = response_substring.encode("utf-8")
self.assertIn(response_substring, body)
returnValue(body)

8
src/allmydata/test/common_util.py
View File

@ -203,6 +203,14 @@ def flip_one_bit(s, offset=0, size=None):
class ReallyEqualMixin(object):
def failUnlessReallyEqual(self, a, b, msg=None):
self.assertEqual(a, b, msg)
# Make sure unicode strings are a consistent type. Specifically there's
# Future newstr (backported Unicode type) vs. Python 2 native unicode
# type. They're equal, and _logically_ the same type, but have
# different types in practice.
if a.__class__ == future_str:
a = unicode(a)
if b.__class__ == future_str:
b = unicode(b)
self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))

10
src/allmydata/test/test_util.py
View File

@ -491,12 +491,16 @@ class JSONBytes(unittest.TestCase):
"""Tests for BytesJSONEncoder."""
def test_encode_bytes(self):
"""BytesJSONEncoder can encode bytes."""
"""BytesJSONEncoder can encode bytes.
Bytes are presumed to be UTF-8 encoded.
"""
snowman = u"def\N{SNOWMAN}\uFF00"
data = {
b"hello": [1, b"cd"],
b"hello": [1, b"cd", {b"abc": [123, snowman.encode("utf-8")]}],
}
expected = {
u"hello": [1, u"cd"],
u"hello": [1, u"cd", {u"abc": [123, snowman]}],
}
# Bytes get passed through as if they were UTF-8 Unicode:
encoded = jsonbytes.dumps(data)

592
src/allmydata/test/web/test_web.py
View File

File diff suppressed because it is too large Load Diff

1
src/allmydata/util/_python3.py
View File

@ -196,5 +196,6 @@ PORTED_TEST_MODULES = [
"allmydata.test.web.test_root",
"allmydata.test.web.test_status",
"allmydata.test.web.test_util",
"allmydata.test.web.test_web",
"allmydata.test.web.test_webish",
]

31
src/allmydata/util/jsonbytes.py
View File

@ -13,20 +13,34 @@ from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import json
def _bytes_to_unicode(obj):
"""Convert any bytes objects to unicode, recursively."""
if isinstance(obj, bytes):
return obj.decode("utf-8")
if isinstance(obj, dict):
new_obj = {}
for k, v in obj.items():
if isinstance(k, bytes):
k = k.decode("utf-8")
v = _bytes_to_unicode(v)
new_obj[k] = v
return new_obj
if isinstance(obj, (list, set, tuple)):
return [_bytes_to_unicode(i) for i in obj]
return obj
class BytesJSONEncoder(json.JSONEncoder):
"""
A JSON encoder than can also encode bytes.
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
def default(self, o):
if isinstance(o, bytes):
return o.decode("utf-8")
return json.JSONEncoder.default(self, o)
def iterencode(self, o, **kwargs):
return json.JSONEncoder.iterencode(self, _bytes_to_unicode(o), **kwargs)
def dumps(obj, *args, **kwargs):
@ -34,13 +48,6 @@ def dumps(obj, *args, **kwargs):
The bytes are assumed to be UTF-8 encoded Unicode strings.
"""
if isinstance(obj, dict):
new_obj = {}
for k, v in obj.items():
if isinstance(k, bytes):
k = k.decode("utf-8")
new_obj[k] = v
obj = new_obj
return json.dumps(obj, cls=BytesJSONEncoder, *args, **kwargs)

2
src/allmydata/web/check_results.py
View File

@ -432,7 +432,7 @@ class DeepCheckResultsRenderer(MultiFormatResource):
return CheckResultsRenderer(self._client,
r.get_results_for_storage_index(si))
except KeyError:
raise WebError("No detailed results for SI %s" % html.escape(name),
raise WebError("No detailed results for SI %s" % html.escape(str(name, "utf-8")),
http.NOT_FOUND)
@render_exception

8
src/allmydata/web/common.py
View File

@ -186,7 +186,7 @@ def convert_children_json(nodemaker, children_json):
children = {}
if children_json:
data = json.loads(children_json)
for (namex, (ctype, propdict)) in data.iteritems():
for (namex, (ctype, propdict)) in data.items():
namex = unicode(namex)
writecap = to_bytes(propdict.get("rw_uri"))
readcap = to_bytes(propdict.get("ro_uri"))
@ -283,8 +283,8 @@ def render_time_attr(t):
# actual exception). The latter is growing increasingly annoying.
def should_create_intermediate_directories(req):
t = get_arg(req, "t", "").strip()
return bool(req.method in ("PUT", "POST") and
t = unicode(get_arg(req, "t", "").strip(), "ascii")
return bool(req.method in (b"PUT", b"POST") and
t not in ("delete", "rename", "rename-form", "check"))
def humanize_exception(exc):
@ -674,7 +674,7 @@ def url_for_string(req, url_string):
and the given URL string.
"""
url = DecodedURL.from_text(url_string.decode("utf-8"))
if url.host == b"":
if not url.host:
root = req.URLPath()
netloc = root.netloc.split(b":", 1)
if len(netloc) == 1:

16
src/allmydata/web/common_py3.py
View File

@ -40,8 +40,12 @@ def get_arg(req, argname, default=None, multiple=False):
results = []
if argname in req.args:
results.extend(req.args[argname])
if req.fields and argname in req.fields:
results.append(req.fields[argname].value)
argname_unicode = unicode(argname, "utf-8")
if req.fields and argname_unicode in req.fields:
value = req.fields[argname_unicode].value
if isinstance(value, unicode):
value = value.encode("utf-8")
results.append(value)
if multiple:
return tuple(results)
if results:
@ -79,7 +83,13 @@ class MultiFormatResource(resource.Resource, object):
if isinstance(t, bytes):
t = unicode(t, "ascii")
renderer = self._get_renderer(t)
return renderer(req)
result = renderer(req)
# On Python 3, json.dumps() returns Unicode for example, but
# twisted.web expects bytes. Instead of updating every single render
# method, just handle Unicode one time here.
if isinstance(result, unicode):
result = result.encode("utf-8")
return result
def _get_renderer(self, fmt):
"""

60
src/allmydata/web/directory.py
View File

@ -1,3 +1,11 @@
"""
TODO: When porting to Python 3, the filename handling logic seems wrong. On
Python 3 filename will _already_ be correctly decoded. So only decode if it's
bytes.
Also there's a lot of code duplication I think.
"""
from past.builtins import unicode
from urllib.parse import quote as url_quote
@ -135,7 +143,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
terminal = (req.prepath + req.postpath)[-1].decode('utf8') == name
nonterminal = not terminal #len(req.postpath) > 0
t = get_arg(req, b"t", b"").strip()
t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
if isinstance(node_or_failure, Failure):
f = node_or_failure
f.trap(NoSuchChildError)
@ -150,10 +158,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
else:
# terminal node
terminal_requests = (
("POST", "mkdir"),
("PUT", "mkdir"),
("POST", "mkdir-with-children"),
("POST", "mkdir-immutable")
(b"POST", "mkdir"),
(b"PUT", "mkdir"),
(b"POST", "mkdir-with-children"),
(b"POST", "mkdir-immutable")
)
if (req.method, t) in terminal_requests:
# final directory
@ -182,8 +190,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
)
return d
leaf_requests = (
("PUT",""),
("PUT","uri"),
(b"PUT",""),
(b"PUT","uri"),
)
if (req.method, t) in leaf_requests:
# we were trying to find the leaf filenode (to put a new
@ -224,7 +232,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
FIXED_OUTPUT_TYPES = ["", "json", "uri", "readonly-uri"]
if not self.node.is_mutable() and t in FIXED_OUTPUT_TYPES:
si = self.node.get_storage_index()
if si and req.setETag('DIR:%s-%s' % (base32.b2a(si), t or "")):
if si and req.setETag(b'DIR:%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
return b""
if not t:
@ -255,7 +263,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
@render_exception
def render_PUT(self, req):
t = get_arg(req, b"t", b"").strip()
t = unicode(get_arg(req, b"t", b"").strip(), "ascii")
replace = parse_replace_arg(get_arg(req, "replace", "true"))
if t == "mkdir":
@ -364,7 +372,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
return d
def _POST_upload(self, req):
charset = get_arg(req, "_charset", "utf-8")
charset = unicode(get_arg(req, "_charset", b"utf-8"), "utf-8")
contents = req.fields["file"]
assert contents.filename is None or isinstance(contents.filename, str)
name = get_arg(req, "name")
@ -374,8 +382,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
if not name:
# this prohibts empty, missing, and all-whitespace filenames
raise WebError("upload requires a name")
assert isinstance(name, str)
name = name.decode(charset)
if isinstance(name, bytes):
name = name.decode(charset)
if "/" in name:
raise WebError("name= may not contain a slash", http.BAD_REQUEST)
assert isinstance(name, unicode)
@ -413,7 +421,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
name = get_arg(req, "name")
if not name:
raise WebError("set-uri requires a name")
charset = get_arg(req, "_charset", "utf-8")
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
name = name.decode(charset)
replace = parse_replace_arg(get_arg(req, "replace", "true"))
@ -436,8 +444,8 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
# a slightly confusing error message if someone does a POST
# without a name= field. For our own HTML this isn't a big
# deal, because we create the 'unlink' POST buttons ourselves.
name = ''
charset = get_arg(req, "_charset", "utf-8")
name = b''
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
name = name.decode(charset)
d = self.node.delete(name)
d.addCallback(lambda res: "thing unlinked")
@ -453,7 +461,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
return self._POST_relink(req)
def _POST_relink(self, req):
charset = get_arg(req, "_charset", "utf-8")
charset = unicode(get_arg(req, "_charset", b"utf-8"), "ascii")
replace = parse_replace_arg(get_arg(req, "replace", "true"))
from_name = get_arg(req, "from_name")
@ -624,14 +632,14 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object):
# TODO test handling of bad JSON
raise
cs = {}
for name, (file_or_dir, mddict) in children.iteritems():
for name, (file_or_dir, mddict) in children.items():
name = unicode(name) # json returns str *or* unicode
writecap = mddict.get('rw_uri')
if writecap is not None:
writecap = str(writecap)
writecap = writecap.encode("utf-8")
readcap = mddict.get('ro_uri')
if readcap is not None:
readcap = str(readcap)
readcap = readcap.encode("utf-8")
cs[name] = (writecap, readcap, mddict.get('metadata'))
d = self.node.set_children(cs, replace)
d.addCallback(lambda res: "Okay so I did it.")
@ -1144,8 +1152,8 @@ def _slashify_path(path):
in it
"""
if not path:
return ""
return "/".join([p.encode("utf-8") for p in path])
return b""
return b"/".join([p.encode("utf-8") for p in path])
def _cap_to_link(root, path, cap):
@ -1234,10 +1242,10 @@ class ManifestResults(MultiFormatResource, ReloadMixin):
req.setHeader("content-type", "text/plain")
lines = []
is_finished = self.monitor.is_finished()
lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
lines.append(b"finished: " + {True: b"yes", False: b"no"}[is_finished])
for path, cap in self.monitor.get_status()["manifest"]:
lines.append(_slashify_path(path) + " " + cap)
return "\n".join(lines) + "\n"
lines.append(_slashify_path(path) + b" " + cap)
return b"\n".join(lines) + b"\n"
def render_JSON(self, req):
req.setHeader("content-type", "text/plain")
@ -1290,7 +1298,7 @@ class DeepSizeResults(MultiFormatResource):
+ stats.get("size-mutable-files", 0)
+ stats.get("size-directories", 0))
output += "size: %d\n" % total
return output
return output.encode("utf-8")
render_TEXT = render_HTML
def render_JSON(self, req):
@ -1315,7 +1323,7 @@ class DeepStatsResults(Resource, object):
req.setHeader("content-type", "text/plain")
s = self.monitor.get_status().copy()
s["finished"] = self.monitor.is_finished()
return json.dumps(s, indent=1)
return json.dumps(s, indent=1).encode("utf-8")
@implementer(IPushProducer)

16
src/allmydata/web/filenode.py
View File

@ -127,7 +127,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
http.NOT_IMPLEMENTED)
if not t:
return self.replace_me_with_a_child(req, self.client, replace)
if t == "uri":
if t == b"uri":
return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % t)
@ -188,8 +188,8 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
# if the client already has the ETag then we can
# short-circuit the whole process.
si = self.node.get_storage_index()
if si and req.setETag('%s-%s' % (base32.b2a(si), t or "")):
return ""
if si and req.setETag(b'%s-%s' % (base32.b2a(si), t.encode("ascii") or b"")):
return b""
if not t:
# just get the contents
@ -281,7 +281,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
assert self.parentnode and self.name
return self.replace_me_with_a_child(req, self.client, replace)
if t == "uri":
if t == b"uri":
if not replace:
raise ExistingChildError()
assert self.parentnode and self.name
@ -309,7 +309,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
assert self.parentnode and self.name
d = self.replace_me_with_a_formpost(req, self.client, replace)
else:
raise WebError("POST to file: bad t=%s" % t)
raise WebError("POST to file: bad t=%s" % unicode(t, "ascii"))
return handle_when_done(req, d)
@ -439,7 +439,7 @@ class FileDownloader(Resource, object):
# bytes we were given in the URL. See the comment in
# FileNodeHandler.render_GET for the sad details.
req.setHeader("content-disposition",
'attachment; filename="%s"' % self.filename)
b'attachment; filename="%s"' % self.filename)
filesize = self.filenode.get_size()
assert isinstance(filesize, (int,long)), filesize
@ -475,8 +475,8 @@ class FileDownloader(Resource, object):
size = contentsize
req.setHeader("content-length", b"%d" % contentsize)
if req.method == "HEAD":
return ""
if req.method == b"HEAD":
return b""
d = self.filenode.read(req, first, size)

11
src/allmydata/web/info.py
View File

@ -1,5 +1,6 @@
import os, urllib
import os
from urllib.parse import quote as urlquote
from twisted.python.filepath import FilePath
from twisted.web.template import tags as T, Element, renderElement, XMLFile, renderer
@ -180,7 +181,7 @@ class MoreInfoElement(Element):
else:
return ""
root = self.get_root(req)
quoted_uri = urllib.quote(node.get_uri())
quoted_uri = urlquote(node.get_uri())
text_plain_url = "%s/file/%s/@@named=/raw.txt" % (root, quoted_uri)
return T.li("Raw data as ", T.a("text/plain", href=text_plain_url))
@ -196,7 +197,7 @@ class MoreInfoElement(Element):
@renderer
def check_form(self, req, tag):
node = self.original
quoted_uri = urllib.quote(node.get_uri())
quoted_uri = urlquote(node.get_uri())
target = self.get_root(req) + "/uri/" + quoted_uri
if IDirectoryNode.providedBy(node):
target += "/"
@ -236,8 +237,8 @@ class MoreInfoElement(Element):
def overwrite_form(self, req, tag):
node = self.original
root = self.get_root(req)
action = "%s/uri/%s" % (root, urllib.quote(node.get_uri()))
done_url = "%s/uri/%s?t=info" % (root, urllib.quote(node.get_uri()))
action = "%s/uri/%s" % (root, urlquote(node.get_uri()))
done_url = "%s/uri/%s?t=info" % (root, urlquote(node.get_uri()))
overwrite = T.form(action=action, method="post",
enctype="multipart/form-data")(
T.fieldset(

7
src/allmydata/web/operations.py
View File

@ -1,3 +1,4 @@
from past.builtins import unicode
import time
from hyperlink import (
@ -101,12 +102,12 @@ class OphandleTable(resource.Resource, service.Service):
def getChild(self, name, req):
ophandle = name
if ophandle not in self.handles:
raise WebError("unknown/expired handle '%s'" % escape(ophandle),
raise WebError("unknown/expired handle '%s'" % escape(unicode(ophandle, "utf-8")),
NOT_FOUND)
(monitor, renderer, when_added) = self.handles[ophandle]
t = get_arg(req, "t", "status")
if t == "cancel" and req.method == "POST":
if t == b"cancel" and req.method == b"POST":
monitor.cancel()
# return the status anyways, but release the handle
self._release_ophandle(ophandle)
@ -151,7 +152,7 @@ class ReloadMixin(object):
@renderer
def refresh(self, req, tag):
if self.monitor.is_finished():
return ""
return b""
tag.attributes["http-equiv"] = "refresh"
tag.attributes["content"] = str(self.REFRESH_TIME)
return tag

15
src/allmydata/web/root.py
View File

@ -1,4 +1,5 @@
from future.utils import PY3
from past.builtins import unicode
import os
import time
@ -97,7 +98,7 @@ class URIHandler(resource.Resource, object):
either "PUT /uri" to create an unlinked file, or
"PUT /uri?t=mkdir" to create an unlinked directory
"""
t = get_arg(req, "t", "").strip()
t = unicode(get_arg(req, "t", "").strip(), "utf-8")
if t == "":
file_format = get_format(req, "CHK")
mutable_type = get_mutable_type(file_format)
@ -120,7 +121,7 @@ class URIHandler(resource.Resource, object):
unlinked file or "POST /uri?t=mkdir" to create a
new directory
"""
t = get_arg(req, "t", "").strip()
t = unicode(get_arg(req, "t", "").strip(), "ascii")
if t in ("", "upload"):
file_format = get_format(req)
mutable_type = get_mutable_type(file_format)
@ -177,7 +178,7 @@ class FileHandler(resource.Resource, object):
@exception_to_child
def getChild(self, name, req):
if req.method not in ("GET", "HEAD"):
if req.method not in (b"GET", b"HEAD"):
raise WebError("/file can only be used with GET or HEAD")
# 'name' must be a file URI
try:
@ -200,7 +201,7 @@ class IncidentReporter(MultiFormatResource):
@render_exception
def render(self, req):
if req.method != "POST":
if req.method != b"POST":
raise WebError("/report_incident can only be used with POST")
log.msg(format="User reports incident through web page: %(details)s",
@ -255,11 +256,11 @@ class Root(MultiFormatResource):
if not path:
# Render "/" path.
return self
if path == "helper_status":
if path == b"helper_status":
# the Helper isn't attached until after the Tub starts, so this child
# needs to created on each request
return status.HelperStatus(self._client.helper)
if path == "storage":
if path == b"storage":
# Storage isn't initialized until after the web hierarchy is
# constructed so this child needs to be created later than
# `__init__`.
@ -293,7 +294,7 @@ class Root(MultiFormatResource):
self._describe_server(server)
for server
in broker.get_known_servers()
))
), key=lambda o: sorted(o.items()))
def _describe_server(self, server):

14
src/allmydata/web/status.py
View File

@ -284,7 +284,7 @@ def _find_overlap(events, start_key, end_key):
rows = []
for ev in events:
ev = ev.copy()
if ev.has_key('server'):
if 'server' in ev:
ev["serverid"] = ev["server"].get_longname()
del ev["server"]
# find an empty slot in the rows
@ -362,8 +362,8 @@ def _find_overlap_requests(events):
def _color(server):
h = hashlib.sha256(server.get_serverid()).digest()
def m(c):
return min(ord(c) / 2 + 0x80, 0xff)
return "#%02x%02x%02x" % (m(h[0]), m(h[1]), m(h[2]))
return min(ord(c) // 2 + 0x80, 0xff)
return "#%02x%02x%02x" % (m(h[0:1]), m(h[1:2]), m(h[2:3]))
class _EventJson(Resource, object):
@ -426,7 +426,7 @@ class DownloadStatusPage(Resource, object):
"""
super(DownloadStatusPage, self).__init__()
self._download_status = download_status
self.putChild("event_json", _EventJson(self._download_status))
self.putChild(b"event_json", _EventJson(self._download_status))
@render_exception
def render_GET(self, req):
@ -1288,14 +1288,14 @@ class Status(MultiFormatResource):
# final URL segment will be an empty string. Resources can
# thus know if they were requested with or without a final
# slash."
if not path and request.postpath != ['']:
if not path and request.postpath != [b'']:
return self
h = self.history
try:
stype, count_s = path.split("-")
stype, count_s = path.split(b"-")
except ValueError:
raise WebError("no '-' in '{}'".format(path))
raise WebError("no '-' in '{}'".format(unicode(path, "utf-8")))
count = int(count_s)
stype = unicode(stype, "ascii")
if stype == "up":

17
src/allmydata/web/unlinked.py
View File

@ -1,5 +1,6 @@
from past.builtins import unicode
import urllib
from urllib.parse import quote as urlquote
from twisted.web import http
from twisted.internet import defer
@ -65,8 +66,8 @@ def POSTUnlinkedCHK(req, client):
# if when_done= is provided, return a redirect instead of our
# usual upload-results page
def _done(upload_results, redir_to):
if "%(uri)s" in redir_to:
redir_to = redir_to.replace("%(uri)s", urllib.quote(upload_results.get_uri()))
if b"%(uri)s" in redir_to:
redir_to = redir_to.replace(b"%(uri)s", urlquote(upload_results.get_uri()).encode("utf-8"))
return url_for_string(req, redir_to)
d.addCallback(_done, when_done)
else:
@ -118,8 +119,8 @@ class UploadResultsElement(status.UploadResultsRendererMixin):
def download_link(self, req, tag):
d = self.upload_results()
d.addCallback(lambda res:
tags.a("/uri/" + res.get_uri(),
href="/uri/" + urllib.quote(res.get_uri())))
tags.a("/uri/" + unicode(res.get_uri(), "utf-8"),
href="/uri/" + urlquote(unicode(res.get_uri(), "utf-8"))))
return d
@ -158,7 +159,7 @@ def POSTUnlinkedCreateDirectory(req, client):
redirect = get_arg(req, "redirect_to_result", "false")
if boolean_of_arg(redirect):
def _then_redir(res):
new_url = "uri/" + urllib.quote(res.get_uri())
new_url = "uri/" + urlquote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
return ''
@ -176,7 +177,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client):
redirect = get_arg(req, "redirect_to_result", "false")
if boolean_of_arg(redirect):
def _then_redir(res):
new_url = "uri/" + urllib.quote(res.get_uri())
new_url = "uri/" + urlquote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
return ''
@ -194,7 +195,7 @@ def POSTUnlinkedCreateImmutableDirectory(req, client):
redirect = get_arg(req, "redirect_to_result", "false")
if boolean_of_arg(redirect):
def _then_redir(res):
new_url = "uri/" + urllib.quote(res.get_uri())
new_url = "uri/" + urlquote(res.get_uri())
req.setResponseCode(http.SEE_OTHER) # 303
req.setHeader('location', new_url)
return ''

44
src/allmydata/webish.py
View File

@ -44,6 +44,43 @@ from .web.storage_plugins import (
StoragePlugins,
)
if PY2:
FileUploadFieldStorage = FieldStorage
else:
class FileUploadFieldStorage(FieldStorage):
"""
Do terrible things to ensure files are still bytes.
On Python 2, uploaded files were always bytes. On Python 3, there's a
heuristic: if the filename is set on a field, it's assumed to be a file
upload and therefore bytes. If no filename is set, it's Unicode.
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
enables setting the filename not via the MIME filename, but via a
separate field called "name".
Thus we need to do this ridiculous workaround. Mypy doesn't like it
either, thus the ``# type: ignore`` below.
Source for idea:
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
"""
@property # type: ignore
def filename(self):
if self.name == "file" and not self._mime_filename:
# We use the file field to upload files, see directory.py's
# _POST_upload. Lack of _mime_filename means we need to trick
# FieldStorage into thinking there is a filename so it'll
# return bytes.
return "unknown-filename"
return self._mime_filename
@filename.setter
def filename(self, value):
self._mime_filename = value
class TahoeLAFSRequest(Request, object):
"""
``TahoeLAFSRequest`` adds several features to a Twisted Web ``Request``
@ -94,7 +131,8 @@ class TahoeLAFSRequest(Request, object):
headers['content-length'] = str(self.content.tell())
self.content.seek(0)
self.fields = FieldStorage(self.content, headers, environ={'REQUEST_METHOD': 'POST'})
self.fields = FileUploadFieldStorage(
self.content, headers, environ={'REQUEST_METHOD': 'POST'})
self.content.seek(0)
self._tahoeLAFSSecurityPolicy()
@ -211,7 +249,7 @@ class WebishServer(service.MultiService):
# use to test ophandle expiration.
self._operations = OphandleTable(clock)
self._operations.setServiceParent(self)
self.root.putChild("operations", self._operations)
self.root.putChild(b"operations", self._operations)
self.root.putChild(b"storage-plugins", StoragePlugins(client))
@ -220,7 +258,7 @@ class WebishServer(service.MultiService):
self.site = TahoeLAFSSite(tempdir, self.root)
self.staticdir = staticdir # so tests can check
if staticdir:
self.root.putChild("static", static.File(staticdir))
self.root.putChild(b"static", static.File(staticdir))
if re.search(r'^\d', webport):
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
# strports must be native strings.

1
tox.ini
View File

@ -114,6 +114,7 @@ commands =
[testenv:typechecks]
basepython = python3
skip_install = True
deps =
mypy