ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-21 10:01:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Trying to get test_grid working on Python 3.

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2020-12-18 11:21:04 -05:00
parent 48b9ffe2a5
commit b24a9f7083
7 changed files with 89 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/allmydata/test/no_network.py
View File

@ -614,7 +614,6 @@ class GridTestMixin(object):
method="GET", clientnum=0, **kwargs):
# if return_response=True, this fires with (data, statuscode,
# respheaders) instead of just data.
assert not isinstance(urlpath, unicode)
url = self.client_baseurls[clientnum] + urlpath
response = yield treq.request(method, url, persistent=False,

123
src/allmydata/test/web/test_grid.py
View File

@ -1,6 +1,7 @@
from __future__ import print_function
import os.path, re, urllib
import os.path, re
from urllib.parse import quote as url_quote
import json
from six.moves import StringIO
@ -37,7 +38,7 @@ DIR_HTML_TAG = '<html lang="en">'
class CompletelyUnhandledError(Exception):
pass
class ErrorBoom(object, resource.Resource):
class ErrorBoom(resource.Resource):
@render_exception
def render(self, req):
raise CompletelyUnhandledError("whoops")
@ -54,25 +55,25 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.set_up_grid()
c0 = self.g.clients[0]
self.uris = {}
DATA = "data" * 100
d = c0.upload(upload.Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(upload.Data(DATA, convergence=b""))
def _stash_uri(ur, which):
self.uris[which] = ur.get_uri()
d.addCallback(_stash_uri, "good")
d.addCallback(lambda ign:
c0.upload(upload.Data(DATA+"1", convergence="")))
c0.upload(upload.Data(DATA+b"1", convergence=b"")))
d.addCallback(_stash_uri, "sick")
d.addCallback(lambda ign:
c0.upload(upload.Data(DATA+"2", convergence="")))
c0.upload(upload.Data(DATA+b"2", convergence=b"")))
d.addCallback(_stash_uri, "dead")
def _stash_mutable_uri(n, which):
self.uris[which] = n.get_uri()
assert isinstance(self.uris[which], str)
assert isinstance(self.uris[which], bytes)
d.addCallback(lambda ign:
c0.create_mutable_file(publish.MutableData(DATA+"3")))
c0.create_mutable_file(publish.MutableData(DATA+b"3")))
d.addCallback(_stash_mutable_uri, "corrupt")
d.addCallback(lambda ign:
c0.upload(upload.Data("literal", convergence="")))
c0.upload(upload.Data("literal", convergence=b"")))
d.addCallback(_stash_uri, "small")
d.addCallback(lambda ign: c0.create_immutable_dirnode({}))
d.addCallback(_stash_mutable_uri, "smalldir")
@ -80,7 +81,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
def _compute_fileurls(ignored):
self.fileurls = {}
for which in self.uris:
self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
self.fileurls[which] = "uri/" + url_quote(self.uris[which])
d.addCallback(_compute_fileurls)
def _clobber_shares(ignored):
@ -203,28 +204,28 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.set_up_grid()
c0 = self.g.clients[0]
self.uris = {}
DATA = "data" * 100
d = c0.upload(upload.Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(upload.Data(DATA, convergence=b""))
def _stash_uri(ur, which):
self.uris[which] = ur.get_uri()
d.addCallback(_stash_uri, "good")
d.addCallback(lambda ign:
c0.upload(upload.Data(DATA+"1", convergence="")))
c0.upload(upload.Data(DATA+b"1", convergence=b"")))
d.addCallback(_stash_uri, "sick")
d.addCallback(lambda ign:
c0.upload(upload.Data(DATA+"2", convergence="")))
c0.upload(upload.Data(DATA+b"2", convergence=b"")))
d.addCallback(_stash_uri, "dead")
def _stash_mutable_uri(n, which):
self.uris[which] = n.get_uri()
assert isinstance(self.uris[which], str)
assert isinstance(self.uris[which], bytes)
d.addCallback(lambda ign:
c0.create_mutable_file(publish.MutableData(DATA+"3")))
c0.create_mutable_file(publish.MutableData(DATA+b"3")))
d.addCallback(_stash_mutable_uri, "corrupt")
def _compute_fileurls(ignored):
self.fileurls = {}
for which in self.uris:
self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
self.fileurls[which] = "uri/" + url_quote(self.uris[which])
d.addCallback(_compute_fileurls)
def _clobber_shares(ignored):
@ -286,8 +287,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.set_up_grid()
c0 = self.g.clients[0]
self.uris = {}
DATA = "data" * 100
d = c0.upload(upload.Data(DATA+"1", convergence=""))
DATA = b"data" * 100
d = c0.upload(upload.Data(DATA+b"1", convergence=b""))
def _stash_uri(ur, which):
self.uris[which] = ur.get_uri()
d.addCallback(_stash_uri, "sick")
@ -295,7 +296,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
def _compute_fileurls(ignored):
self.fileurls = {}
for which in self.uris:
self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
self.fileurls[which] = "uri/" + url_quote(self.uris[which])
d.addCallback(_compute_fileurls)
def _clobber_shares(ignored):
@ -329,7 +330,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.fileurls = {}
# the future cap format may contain slashes, which must be tolerated
expected_info_url = "uri/%s?t=info" % urllib.quote(unknown_rwcap,
expected_info_url = "uri/%s?t=info" % url_quote(unknown_rwcap,
safe="")
if immutable:
@ -343,8 +344,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
def _stash_root_and_create_file(n):
self.rootnode = n
self.rooturl = "uri/" + urllib.quote(n.get_uri())
self.rourl = "uri/" + urllib.quote(n.get_readonly_uri())
self.rooturl = "uri/" + url_quote(n.get_uri())
self.rourl = "uri/" + url_quote(n.get_readonly_uri())
if not immutable:
return self.rootnode.set_node(name, future_node)
d.addCallback(_stash_root_and_create_file)
@ -510,7 +511,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.failUnlessIn("CHK", cap.to_string())
self.cap = cap
self.rootnode = dn
self.rooturl = "uri/" + urllib.quote(dn.get_uri())
self.rooturl = "uri/" + url_quote(dn.get_uri())
return download_to_data(dn._node)
d.addCallback(_created)
@ -559,7 +560,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
if td.text != u"FILE":
continue
a = td.findNextSibling()(u"a")[0]
self.assertIn(urllib.quote(lonely_uri), a[u"href"])
self.assertIn(url_quote(lonely_uri), a[u"href"])
self.assertEqual(u"lonely", a.text)
self.assertEqual(a[u"rel"], [u"noreferrer"])
self.assertEqual(u"{}".format(len("one")), td.findNextSibling().findNextSibling().text)
@ -573,7 +574,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
if a.text == u"More Info"
)
self.assertEqual(1, len(infos))
self.assertTrue(infos[0].endswith(urllib.quote(lonely_uri) + "?t=info"))
self.assertTrue(infos[0].endswith(url_quote(lonely_uri) + "?t=info"))
d.addCallback(_check_html)
# ... and in JSON.
@ -596,12 +597,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
c0 = self.g.clients[0]
self.uris = {}
self.fileurls = {}
DATA = "data" * 100
DATA = b"data" * 100
d = c0.create_dirnode()
def _stash_root_and_create_file(n):
self.rootnode = n
self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
return n.add_file(u"good", upload.Data(DATA, convergence=""))
self.fileurls["root"] = "uri/" + url_quote(n.get_uri())
return n.add_file(u"good", upload.Data(DATA, convergence=b""))
d.addCallback(_stash_root_and_create_file)
def _stash_uri(fn, which):
self.uris[which] = fn.get_uri()
@ -610,12 +611,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
d.addCallback(lambda ign:
self.rootnode.add_file(u"small",
upload.Data("literal",
convergence="")))
convergence=b"")))
d.addCallback(_stash_uri, "small")
d.addCallback(lambda ign:
self.rootnode.add_file(u"sick",
upload.Data(DATA+"1",
convergence="")))
upload.Data(DATA+b"1",
convergence=b"")))
d.addCallback(_stash_uri, "sick")
# this tests that deep-check and stream-manifest will ignore
@ -695,8 +696,8 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
d.addCallback(_stash_uri, "subdir")
d.addCallback(lambda subdir_node:
subdir_node.add_file(u"grandchild",
upload.Data(DATA+"2",
convergence="")))
upload.Data(DATA+b"2",
convergence=b"")))
d.addCallback(_stash_uri, "grandchild")
d.addCallback(lambda ign:
@ -770,12 +771,12 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
c0 = self.g.clients[0]
self.uris = {}
self.fileurls = {}
DATA = "data" * 100
DATA = b"data" * 100
d = c0.create_dirnode()
def _stash_root_and_create_file(n):
self.rootnode = n
self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
return n.add_file(u"good", upload.Data(DATA, convergence=""))
self.fileurls["root"] = "uri/" + url_quote(n.get_uri())
return n.add_file(u"good", upload.Data(DATA, convergence=b""))
d.addCallback(_stash_root_and_create_file)
def _stash_uri(fn, which):
self.uris[which] = fn.get_uri()
@ -783,17 +784,17 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
d.addCallback(lambda ign:
self.rootnode.add_file(u"small",
upload.Data("literal",
convergence="")))
convergence=b"")))
d.addCallback(_stash_uri, "small")
d.addCallback(lambda ign:
self.rootnode.add_file(u"sick",
upload.Data(DATA+"1",
convergence="")))
upload.Data(DATA+b"1",
convergence=b"")))
d.addCallback(_stash_uri, "sick")
#d.addCallback(lambda ign:
# self.rootnode.add_file(u"dead",
# upload.Data(DATA+"2",
# convergence="")))
# upload.Data(DATA+b"2",
# convergence=b"")))
#d.addCallback(_stash_uri, "dead")
#d.addCallback(lambda ign: c0.create_mutable_file("mutable"))
@ -888,25 +889,25 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
self.set_up_grid(num_clients=2, oneshare=True)
c0 = self.g.clients[0]
self.uris = {}
DATA = "data" * 100
d = c0.upload(upload.Data(DATA, convergence=""))
DATA = b"data" * 100
d = c0.upload(upload.Data(DATA, convergence=b""))
def _stash_uri(ur, which):
self.uris[which] = ur.get_uri()
d.addCallback(_stash_uri, "one")
d.addCallback(lambda ign:
c0.upload(upload.Data(DATA+"1", convergence="")))
c0.upload(upload.Data(DATA+b"1", convergence=b"")))
d.addCallback(_stash_uri, "two")
def _stash_mutable_uri(n, which):
self.uris[which] = n.get_uri()
assert isinstance(self.uris[which], str)
assert isinstance(self.uris[which], bytes)
d.addCallback(lambda ign:
c0.create_mutable_file(publish.MutableData(DATA+"2")))
c0.create_mutable_file(publish.MutableData(DATA+b"2")))
d.addCallback(_stash_mutable_uri, "mutable")
def _compute_fileurls(ignored):
self.fileurls = {}
for which in self.uris:
self.fileurls[which] = "uri/" + urllib.quote(self.uris[which])
self.fileurls[which] = "uri/" + url_quote(self.uris[which])
d.addCallback(_compute_fileurls)
d.addCallback(self._count_leases, "one")
@ -982,13 +983,13 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
c0 = self.g.clients[0]
self.uris = {}
self.fileurls = {}
DATA = "data" * 100
DATA = b"data" * 100
d = c0.create_dirnode()
def _stash_root_and_create_file(n):
self.rootnode = n
self.uris["root"] = n.get_uri()
self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
return n.add_file(u"one", upload.Data(DATA, convergence=""))
self.fileurls["root"] = "uri/" + url_quote(n.get_uri())
return n.add_file(u"one", upload.Data(DATA, convergence=b""))
d.addCallback(_stash_root_and_create_file)
def _stash_uri(fn, which):
self.uris[which] = fn.get_uri()
@ -996,7 +997,7 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
d.addCallback(lambda ign:
self.rootnode.add_file(u"small",
upload.Data("literal",
convergence="")))
convergence=b"")))
d.addCallback(_stash_uri, "small")
d.addCallback(lambda ign:
@ -1051,34 +1052,34 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
c0 = self.g.clients[0]
c0.encoding_params['happy'] = 2
self.fileurls = {}
DATA = "data" * 100
DATA = b"data" * 100
d = c0.create_dirnode()
def _stash_root(n):
self.fileurls["root"] = "uri/" + urllib.quote(n.get_uri())
self.fileurls["root"] = "uri/" + url_quote(n.get_uri())
self.fileurls["imaginary"] = self.fileurls["root"] + "/imaginary"
return n
d.addCallback(_stash_root)
d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence="")))
d.addCallback(lambda ign: c0.upload(upload.Data(DATA, convergence=b"")))
def _stash_bad(ur):
self.fileurls["1share"] = "uri/" + urllib.quote(ur.get_uri())
self.fileurls["1share"] = "uri/" + url_quote(ur.get_uri())
self.delete_shares_numbered(ur.get_uri(), range(1,10))
u = uri.from_string(ur.get_uri())
u.key = testutil.flip_bit(u.key, 0)
baduri = u.to_string()
self.fileurls["0shares"] = "uri/" + urllib.quote(baduri)
self.fileurls["0shares"] = "uri/" + url_quote(baduri)
d.addCallback(_stash_bad)
d.addCallback(lambda ign: c0.create_dirnode())
def _mangle_dirnode_1share(n):
u = n.get_uri()
url = self.fileurls["dir-1share"] = "uri/" + urllib.quote(u)
url = self.fileurls["dir-1share"] = "uri/" + url_quote(u)
self.fileurls["dir-1share-json"] = url + "?t=json"
self.delete_shares_numbered(u, range(1,10))
d.addCallback(_mangle_dirnode_1share)
d.addCallback(lambda ign: c0.create_dirnode())
def _mangle_dirnode_0share(n):
u = n.get_uri()
url = self.fileurls["dir-0share"] = "uri/" + urllib.quote(u)
url = self.fileurls["dir-0share"] = "uri/" + url_quote(u)
self.fileurls["dir-0share-json"] = url + "?t=json"
self.delete_shares_numbered(u, range(0,10))
d.addCallback(_mangle_dirnode_0share)
@ -1269,9 +1270,9 @@ class Grid(GridTestMixin, WebErrorMixin, ShouldFailMixin, testutil.ReallyEqualMi
c0 = self.g.clients[0]
fn = c0.config.get_config_path("access.blacklist")
self.uris = {}
DATA = "off-limits " * 50
DATA = b"off-limits " * 50
d = c0.upload(upload.Data(DATA, convergence=""))
d = c0.upload(upload.Data(DATA, convergence=b""))
def _stash_uri_and_create_dir(ur):
self.uri = ur.get_uri()
self.url = "uri/"+self.uri

9
src/allmydata/web/common.py
View File

@ -1,4 +1,5 @@
from past.builtins import unicode
from six import ensure_text
import time
import json
@ -99,11 +100,13 @@ def get_filenode_metadata(filenode):
def boolean_of_arg(arg):
# TODO: ""
arg = ensure_text(arg)
if arg.lower() not in ("true", "t", "1", "false", "f", "0", "on", "off"):
raise WebError("invalid boolean argument: %r" % (arg,), http.BAD_REQUEST)
return arg.lower() in ("true", "t", "1", "on")
def parse_replace_arg(replace):
replace = ensure_text(replace)
if replace.lower() == "only-files":
return replace
try:
@ -118,11 +121,11 @@ def get_format(req, default="CHK"):
if boolean_of_arg(get_arg(req, "mutable", "false")):
return "SDMF"
return default
if arg.upper() == "CHK":
if arg.upper() == b"CHK":
return "CHK"
elif arg.upper() == "SDMF":
elif arg.upper() == b"SDMF":
return "SDMF"
elif arg.upper() == "MDMF":
elif arg.upper() == b"MDMF":
return "MDMF"
else:
raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg,

8
src/allmydata/web/common_py3.py
View File

@ -4,6 +4,8 @@ Common utilities that are available from Python 3.
Can eventually be merged back into allmydata.web.common.
"""
from past.builtins import unicode
from twisted.web import resource, http
from allmydata.util import abbreviate
@ -23,7 +25,13 @@ def get_arg(req, argname, default=None, multiple=False):
empty), starting with all those in the query args.
:param TahoeLAFSRequest req: The request to consider.
:return: Either bytes or tuple of bytes.
"""
if isinstance(argname, unicode):
argname = argname.encode("ascii")
if isinstance(default, unicode):
default = default.encode("ascii")
results = []
if argname in req.args:
results.extend(req.args[argname])

14
src/allmydata/web/filenode.py
View File

@ -133,9 +133,9 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
@render_exception
def render_POST(self, req):
t = get_arg(req, "t", "").strip()
replace = boolean_of_arg(get_arg(req, "replace", "true"))
if t == "upload":
t = get_arg(req, b"t", b"").strip()
replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
if t == b"upload":
# like PUT, but get the file data from an HTML form's input field.
# We could get here from POST /uri/mutablefilecap?t=upload,
# or POST /uri/path/file?t=upload, or
@ -290,11 +290,11 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
@render_exception
def render_POST(self, req):
t = get_arg(req, "t", "").strip()
replace = boolean_of_arg(get_arg(req, "replace", "true"))
if t == "check":
t = get_arg(req, b"t", b"").strip()
replace = boolean_of_arg(get_arg(req, b"replace", b"true"))
if t == b"check":
d = self._POST_check(req)
elif t == "upload":
elif t == b"upload":
# like PUT, but get the file data from an HTML form's input field
# We could get here from POST /uri/mutablefilecap?t=upload,
# or POST /uri/path/file?t=upload, or

5
src/allmydata/web/root.py
View File

@ -226,7 +226,10 @@ class Root(MultiFormatResource):
self._client = client
self._now_fn = now_fn
self.putChild("uri", URIHandler(client))
# Children need to be bytes; for now just doing these to make specific
# tests pass on Python 3, but eventually will do all them when this
# module is ported to Python 3 (if not earlier).
self.putChild(b"uri", URIHandler(client))
self.putChild("cap", URIHandler(client))
# Handler for everything beneath "/private", an area of the resource

3
src/allmydata/webish.py
View File

@ -110,6 +110,7 @@ def _get_client_ip(request):
def _logFormatter(logDateTime, request):
print("REQUEST: {}".format(request.uri))
# we build up a log string that hides most of the cap, to preserve
# user privacy. We retain the query args so we can identify things
# like t=json. Then we send it to the flog. We make no attempt to
@ -128,7 +129,7 @@ def _logFormatter(logDateTime, request):
# sure we censor these too.
if queryargs.startswith(b"uri="):
queryargs = b"uri=[CENSORED]"
queryargs = "?" + queryargs
queryargs = b"?" + queryargs
if path.startswith(b"/uri/"):
path = b"/uri/[CENSORED]"
elif path.startswith(b"/file/"):