ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-29 17:28:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #978 from tahoe-lafs/3606.test_system-web-python3

Browse Source 
Port more of test_system to Python3

Fixes ticket:3606
This commit is contained in:
Itamar Turner-Trauring 2021-02-15 11:38:02 -05:00 committed by GitHub
commit 2aec4756b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 51 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3606.minor Normal file
View File

12
src/allmydata/control.py
View File

@ -65,7 +65,7 @@ class ControlServer(Referenceable, service.Service):
tempdir = tempfile.mkdtemp()
filename = os.path.join(tempdir, "data")
f = open(filename, "wb")
block = "a" * 8192
block = b"a" * 8192
while size > 0:
l = min(size, 8192)
f.write(block[:l])
@ -126,7 +126,7 @@ class ControlServer(Referenceable, service.Service):
server_name = server.get_longname()
storage_server = server.get_storage_server()
start = time.time()
d = storage_server.get_buckets("\x00" * 16)
d = storage_server.get_buckets(b"\x00" * 16)
def _done(ignored):
stop = time.time()
elapsed = stop - start
@ -138,7 +138,7 @@ class ControlServer(Referenceable, service.Service):
d.addCallback(self._do_one_ping, everyone_left, results)
def _average(res):
averaged = {}
for server_name,times in results.iteritems():
for server_name,times in results.items():
averaged[server_name] = sum(times) / len(times)
return averaged
d.addCallback(_average)
@ -168,19 +168,19 @@ class SpeedTest(object):
fn = os.path.join(self.basedir, str(i))
if os.path.exists(fn):
os.unlink(fn)
f = open(fn, "w")
f = open(fn, "wb")
f.write(os.urandom(8))
s -= 8
while s > 0:
chunk = min(s, 4096)
f.write("\x00" * chunk)
f.write(b"\x00" * chunk)
s -= chunk
f.close()
def do_upload(self):
d = defer.succeed(None)
def _create_slot(res):
d1 = self.parent.create_mutable_file("")
d1 = self.parent.create_mutable_file(b"")
def _created(n):
self._n = n
d1.addCallback(_created)

2
src/allmydata/interfaces.py
View File

@ -2858,7 +2858,7 @@ class RIControlClient(RemoteInterface):
@return: a dictionary mapping peerid to a float (RTT time in seconds)
"""
return DictOf(str, float)
return DictOf(bytes, float)
UploadResults = Any() #DictOf(bytes, bytes)

2
src/allmydata/scripts/debug.py
View File

@ -645,7 +645,7 @@ def find_shares(options):
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
out = options.stdout
sharedir = storage_index_to_dir(si_a2b(options.si_s))
sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8")))
for d in options.nodedirs:
d = os.path.join(d, "storage", "shares", sharedir)
if os.path.exists(d):

6
src/allmydata/stats.py
View File

@ -17,7 +17,7 @@ from twisted.application.internet import TimerService
from zope.interface import implementer
from foolscap.api import eventually
from allmydata.util import log
from allmydata.util import log, dictutil
from allmydata.interfaces import IStatsProducer
@implementer(IStatsProducer)
@ -79,15 +79,13 @@ class StatsProvider(service.MultiService):
service.MultiService.__init__(self)
self.node = node
self.counters = {}
self.counters = dictutil.UnicodeKeyDict()
self.stats_producers = []
self.cpu_monitor = CPUUsageMonitor()
self.cpu_monitor.setServiceParent(self)
self.register_producer(self.cpu_monitor)
def count(self, name, delta=1):
if isinstance(name, str):
name = name.encode("utf-8")
val = self.counters.setdefault(name, 0)
self.counters[name] = val + delta

2
src/allmydata/test/mutable/test_version.py
View File

@ -86,7 +86,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
def _debug(n):
fso = debug.FindSharesOptions()
storage_index = base32.b2a(n.get_storage_index())
fso.si_s = storage_index
fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3
fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir)))
for (i,ss,storedir)
in self.iterate_servers()]

33
src/allmydata/test/test_system.py
View File

@ -1324,9 +1324,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
s = stats["stats"]
self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1)
c = stats["counters"]
# Probably this should be Unicode eventually? But we haven't ported
# stats code yet.
self.failUnless(b"storage_server.allocate" in c)
self.failUnless("storage_server.allocate" in c)
d.addCallback(_grab_stats)
return d
@ -1631,7 +1629,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
# the key, which should cause the download to fail the post-download
# plaintext_hash check.
@skipIf(PY3, "Python 3 web support hasn't happened yet.")
def test_filesystem(self):
self.basedir = "system/SystemTest/test_filesystem"
self.data = LARGE_DATA
@ -1669,7 +1666,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(self.log, "did _check_publish_private")
d.addCallback(self._test_web)
d.addCallback(self._test_control)
d.addCallback(self._test_cli)
if PY2:
# TODO when CLI is ported to Python 3, reenable.
d.addCallback(self._test_cli)
# P now has four top-level children:
# P/personal/sekrit data
# P/s2-ro/
@ -1923,9 +1922,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
if isinstance(value, tuple):
filename, value = value
form.append(b'Content-Disposition: form-data; name="%s"; '
b'filename="%s"' % (name, filename.encode("utf-8")))
b'filename="%s"' % (name.encode("utf-8"), filename.encode("utf-8")))
else:
form.append(b'Content-Disposition: form-data; name="%s"' % name)
form.append(b'Content-Disposition: form-data; name="%s"' % name.encode("utf-8"))
form.append(b'')
form.append(b"%s" % (value,))
form.append(sep)
@ -1982,22 +1981,22 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(self.log, "done with _got_subdir1")
d.addCallback(lambda res: self.GET(public + "/subdir1/mydata567"))
def _got_data(page):
self.failUnlessEqual(page, self.data)
self.failUnlessEqual(page.encode("utf-8"), self.data)
d.addCallback(_got_data)
# download from a URI embedded in a URL
d.addCallback(self.log, "_get_from_uri")
def _get_from_uri(res):
return self.GET("uri/%s?filename=%s" % (self.uri, "mydata567"))
return self.GET("uri/%s?filename=%s" % (str(self.uri, "utf-8"), "mydata567"))
d.addCallback(_get_from_uri)
def _got_from_uri(page):
self.failUnlessEqual(page, self.data)
self.failUnlessEqual(page.encode("utf-8"), self.data)
d.addCallback(_got_from_uri)
# download from a URI embedded in a URL, second form
d.addCallback(self.log, "_get_from_uri2")
def _get_from_uri2(res):
return self.GET("uri?uri=%s" % (self.uri,))
return self.GET("uri?uri=%s" % (str(self.uri, "utf-8"),))
d.addCallback(_get_from_uri2)
d.addCallback(_got_from_uri)
@ -2006,9 +2005,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
@defer.inlineCallbacks
def _get_from_bogus_uri(res):
d1 = self.GET("uri/%s?filename=%s"
% (self.mangle_uri(self.uri), "mydata567"))
% (str(self.mangle_uri(self.uri), "utf-8"), "mydata567"))
e = yield self.assertFailure(d1, Error)
self.assertEquals(e.status, "410")
self.assertEquals(e.status, b"410")
d.addCallback(_get_from_bogus_uri)
d.addCallback(self.log, "_got_from_bogus_uri", level=log.UNUSUAL)
@ -2092,14 +2091,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
workdir = os.path.join(self.getdir("client0"), "helper")
incfile = os.path.join(workdir, "CHK_incoming", "spurious")
f = open(incfile, "wb")
f.write("small file")
f.write(b"small file")
f.close()
then = time.time() - 86400*3
now = time.time()
os.utime(incfile, (now, then))
encfile = os.path.join(workdir, "CHK_encoding", "spurious")
f = open(encfile, "wb")
f.write("less small file")
f.write(b"less small file")
f.close()
os.utime(encfile, (now, then))
d.addCallback(_got_helper_status)
@ -2140,7 +2139,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(lambda res: self.GET("statistics"))
def _got_stats(res):
self.failUnlessIn("Operational Statistics", res)
self.failUnlessIn(" 'downloader.files_downloaded': 5,", res)
self.failUnlessIn(' "downloader.files_downloaded": 5,', res)
d.addCallback(_got_stats)
d.addCallback(lambda res: self.GET("statistics?t=json"))
def _got_stats_json(res):
@ -2348,7 +2347,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
for i in range(10):
fn = os.path.join(self.basedir, "file%d" % i)
files.append(fn)
data = "data to be uploaded: file%d\n" % i
data = b"data to be uploaded: file%d\n" % i
datas.append(data)
with open(fn, "wb") as f:
f.write(data)

26
src/allmydata/test/web/test_web.py
View File

@ -1760,7 +1760,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_PUT_NEWFILEURL_unlinked_bad_format(self):
contents = self.NEWFILE_CONTENTS * 300000
yield self.assertHTTPError(self.webish_url + "/uri?format=foo", 400,
"Unknown format:",
"Unknown format: foo",
method="put", data=contents)
def test_PUT_NEWFILEURL_range_bad(self):
@ -1813,7 +1813,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_PUT_NEWFILEURL_bad_t(self):
d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request",
"PUT to a file: bad t=",
"PUT to a file: bad t=bogus",
self.PUT, self.public_url + "/foo/bar.txt?t=bogus",
b"contents")
return d
@ -2344,7 +2344,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_PUT_NEWDIRURL_bad_format(self):
url = (self.webish_url + self.public_url +
"/foo/newdir=?t=mkdir&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="put", data="")
def test_POST_NEWDIRURL(self):
@ -2377,7 +2377,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_POST_NEWDIRURL_bad_format(self):
url = (self.webish_url + self.public_url +
"/foo/newdir?t=mkdir&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post", data="")
def test_POST_NEWDIRURL_emptyname(self):
@ -2454,7 +2454,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
(newkids, caps) = self._create_initial_children()
url = (self.webish_url + self.public_url +
"/foo/newdir?t=mkdir-with-children&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post", data=json.dumps(newkids).encode("utf-8"))
def test_POST_NEWDIRURL_immutable(self):
@ -2578,7 +2578,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_PUT_NEWDIRURL_mkdirs_bad_format(self):
url = (self.webish_url + self.public_url +
"/foo/subdir/newdir?t=mkdir&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="put", data="")
def test_DELETE_DIRURL(self):
@ -2857,7 +2857,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
url = self.webish_url + "/uri?t=upload&format=foo"
body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
yield self.assertHTTPError(url, 400,
"Unknown format:",
"Unknown format: foo",
method="post", data=body, headers=headers)
def test_POST_upload_format(self):
@ -2892,7 +2892,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_POST_upload_bad_format(self):
url = self.webish_url + self.public_url + "/foo?t=upload&format=foo"
body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000))
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post", data=body, headers=headers)
def test_POST_upload_mutable(self):
@ -3388,7 +3388,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_POST_mkdir_bad_format(self):
url = (self.webish_url + self.public_url +
"/foo?t=mkdir&name=newdir&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post")
def test_POST_mkdir_initial_children(self):
@ -3440,7 +3440,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
(newkids, caps) = self._create_initial_children()
url = (self.webish_url + self.public_url +
"/foo?t=mkdir-with-children&name=newdir&format=foo")
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post", data=json.dumps(newkids).encode("utf-8"))
def test_POST_mkdir_immutable(self):
@ -3519,7 +3519,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
@inlineCallbacks
def test_POST_mkdir_no_parentdir_noredirect_bad_format(self):
url = self.webish_url + self.public_url + "/uri?t=mkdir&format=foo"
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="post")
def test_POST_mkdir_no_parentdir_noredirect2(self):
@ -4462,7 +4462,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
def test_PUT_NEWFILEURL_bad_format(self):
new_contents = self.NEWFILE_CONTENTS * 300000
url = self.webish_url + self.public_url + "/foo/foo.txt?format=foo"
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="put", data=new_contents)
def test_PUT_NEWFILEURL_uri_replace(self):
@ -4595,7 +4595,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
@inlineCallbacks
def test_PUT_mkdir_bad_format(self):
url = self.webish_url + "/uri?t=mkdir&format=foo"
yield self.assertHTTPError(url, 400, "Unknown format:",
yield self.assertHTTPError(url, 400, "Unknown format: foo",
method="put", data=b"")
def test_POST_check(self):

6
src/allmydata/util/_python3.py
View File

@ -180,9 +180,9 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_storage_client",
"allmydata.test.test_storage_web",
# Only partially ported, test_filesystem_with_cli_in_subprocess and
# test_filesystem methods aren't ported yet, should be done once CLI and
# web are ported respectively.
# Only partially ported, test_filesystem_with_cli_in_subprocess isn't
# ported yet, nor is part of test_filesystem (the call to _test_cli). This
# should be done once CLI is ported.
"allmydata.test.test_system",
"allmydata.test.test_time_format",

2
src/allmydata/web/common.py
View File

@ -148,7 +148,7 @@ def get_format(req, default="CHK"):
elif arg.upper() == b"MDMF":
return "MDMF"
else:
raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg,
raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % ensure_str(arg),
http.BAD_REQUEST)
def get_mutable_type(file_format): # accepts result of get_format()

5
src/allmydata/web/filenode.py
View File

@ -1,4 +1,5 @@
from past.builtins import unicode, long
from six import ensure_str
from twisted.web import http, static
from twisted.internet import defer
@ -130,7 +131,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin):
if t == b"uri":
return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % t)
raise WebError("PUT to a file: bad t=%s" % ensure_str(t))
@render_exception
def render_POST(self, req):
@ -287,7 +288,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object):
assert self.parentnode and self.name
return self.replace_me_with_a_childcap(req, self.client, replace)
raise WebError("PUT to a file: bad t=%s" % t)
raise WebError("PUT to a file: bad t=%s" % ensure_str(t))
@render_exception
def render_POST(self, req):

2
src/allmydata/web/operations.py
View File

@ -152,7 +152,7 @@ class ReloadMixin(object):
@renderer
def refresh(self, req, tag):
if self.monitor.is_finished():
return b""
return ""
tag.attributes["http-equiv"] = "refresh"
tag.attributes["content"] = str(self.REFRESH_TIME)
return tag

7
src/allmydata/web/status.py
View File

@ -1,6 +1,5 @@
from past.builtins import long, unicode
import pprint
import itertools
import hashlib
from twisted.internet import defer
@ -61,7 +60,7 @@ class UploadResultsRendererMixin(Element):
return "None"
ul = tags.ul()
for shnum, servers in sorted(sharemap.items()):
server_names = ', '.join([s.get_name() for s in servers])
server_names = ', '.join([unicode(s.get_name(), "utf-8") for s in servers])
ul(tags.li("%d -> placed on [%s]" % (shnum, server_names)))
return ul
d.addCallback(_render)
@ -75,7 +74,7 @@ class UploadResultsRendererMixin(Element):
if servermap is None:
return "None"
ul = tags.ul()
for server, shnums in sorted(servermap.items()):
for server, shnums in sorted(servermap.items(), key=id):
shares_s = ",".join(["#%d" % shnum for shnum in shnums])
ul(tags.li("[%s] got share%s: %s" % (server.get_name(),
plural(shnums), shares_s)))
@ -1595,5 +1594,5 @@ class StatisticsElement(Element):
@renderer
def raw(self, req, tag):
raw = pprint.pformat(self._stats)
raw = json.dumps(self._stats, sort_keys=True, indent=4)
return tag(raw)