diff --git a/newsfragments/3606.minor b/newsfragments/3606.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/control.py b/src/allmydata/control.py index b4603b2ac..25ce3cea7 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -65,7 +65,7 @@ class ControlServer(Referenceable, service.Service): tempdir = tempfile.mkdtemp() filename = os.path.join(tempdir, "data") f = open(filename, "wb") - block = "a" * 8192 + block = b"a" * 8192 while size > 0: l = min(size, 8192) f.write(block[:l]) @@ -126,7 +126,7 @@ class ControlServer(Referenceable, service.Service): server_name = server.get_longname() storage_server = server.get_storage_server() start = time.time() - d = storage_server.get_buckets("\x00" * 16) + d = storage_server.get_buckets(b"\x00" * 16) def _done(ignored): stop = time.time() elapsed = stop - start @@ -138,7 +138,7 @@ class ControlServer(Referenceable, service.Service): d.addCallback(self._do_one_ping, everyone_left, results) def _average(res): averaged = {} - for server_name,times in results.iteritems(): + for server_name,times in results.items(): averaged[server_name] = sum(times) / len(times) return averaged d.addCallback(_average) @@ -168,19 +168,19 @@ class SpeedTest(object): fn = os.path.join(self.basedir, str(i)) if os.path.exists(fn): os.unlink(fn) - f = open(fn, "w") + f = open(fn, "wb") f.write(os.urandom(8)) s -= 8 while s > 0: chunk = min(s, 4096) - f.write("\x00" * chunk) + f.write(b"\x00" * chunk) s -= chunk f.close() def do_upload(self): d = defer.succeed(None) def _create_slot(res): - d1 = self.parent.create_mutable_file("") + d1 = self.parent.create_mutable_file(b"") def _created(n): self._n = n d1.addCallback(_created) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 0dd5ddc83..96d3e813c 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -2858,7 +2858,7 @@ class RIControlClient(RemoteInterface): @return: a dictionary mapping peerid to a float (RTT time in seconds) """ - return DictOf(str, float) + return DictOf(bytes, float) UploadResults = Any() #DictOf(bytes, bytes) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 550c37fde..e4bcf165e 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -645,7 +645,7 @@ def find_shares(options): from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path out = options.stdout - sharedir = storage_index_to_dir(si_a2b(options.si_s)) + sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8"))) for d in options.nodedirs: d = os.path.join(d, "storage", "shares", sharedir) if os.path.exists(d): diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 91205a93c..7137ba28e 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -17,7 +17,7 @@ from twisted.application.internet import TimerService from zope.interface import implementer from foolscap.api import eventually -from allmydata.util import log +from allmydata.util import log, dictutil from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) @@ -79,15 +79,13 @@ class StatsProvider(service.MultiService): service.MultiService.__init__(self) self.node = node - self.counters = {} + self.counters = dictutil.UnicodeKeyDict() self.stats_producers = [] self.cpu_monitor = CPUUsageMonitor() self.cpu_monitor.setServiceParent(self) self.register_producer(self.cpu_monitor) def count(self, name, delta=1): - if isinstance(name, str): - name = name.encode("utf-8") val = self.counters.setdefault(name, 0) self.counters[name] = val + delta diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py index 06191b5fe..042305c24 100644 --- a/src/allmydata/test/mutable/test_version.py +++ b/src/allmydata/test/mutable/test_version.py @@ -86,7 +86,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \ def _debug(n): fso = debug.FindSharesOptions() storage_index = base32.b2a(n.get_storage_index()) - fso.si_s = storage_index + fso.si_s = str(storage_index, "utf-8") # command-line options are unicode on Python 3 fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir))) for (i,ss,storedir) in self.iterate_servers()] diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index bf115f127..8214c312c 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1324,9 +1324,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): s = stats["stats"] self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1) c = stats["counters"] - # Probably this should be Unicode eventually? But we haven't ported - # stats code yet. - self.failUnless(b"storage_server.allocate" in c) + self.failUnless("storage_server.allocate" in c) d.addCallback(_grab_stats) return d @@ -1631,7 +1629,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # the key, which should cause the download to fail the post-download # plaintext_hash check. - @skipIf(PY3, "Python 3 web support hasn't happened yet.") def test_filesystem(self): self.basedir = "system/SystemTest/test_filesystem" self.data = LARGE_DATA @@ -1669,7 +1666,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) d.addCallback(self._test_control) - d.addCallback(self._test_cli) + if PY2: + # TODO when CLI is ported to Python 3, reenable. + d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data # P/s2-ro/ @@ -1923,9 +1922,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if isinstance(value, tuple): filename, value = value form.append(b'Content-Disposition: form-data; name="%s"; ' - b'filename="%s"' % (name, filename.encode("utf-8"))) + b'filename="%s"' % (name.encode("utf-8"), filename.encode("utf-8"))) else: - form.append(b'Content-Disposition: form-data; name="%s"' % name) + form.append(b'Content-Disposition: form-data; name="%s"' % name.encode("utf-8")) form.append(b'') form.append(b"%s" % (value,)) form.append(sep) @@ -1982,22 +1981,22 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "done with _got_subdir1") d.addCallback(lambda res: self.GET(public + "/subdir1/mydata567")) def _got_data(page): - self.failUnlessEqual(page, self.data) + self.failUnlessEqual(page.encode("utf-8"), self.data) d.addCallback(_got_data) # download from a URI embedded in a URL d.addCallback(self.log, "_get_from_uri") def _get_from_uri(res): - return self.GET("uri/%s?filename=%s" % (self.uri, "mydata567")) + return self.GET("uri/%s?filename=%s" % (str(self.uri, "utf-8"), "mydata567")) d.addCallback(_get_from_uri) def _got_from_uri(page): - self.failUnlessEqual(page, self.data) + self.failUnlessEqual(page.encode("utf-8"), self.data) d.addCallback(_got_from_uri) # download from a URI embedded in a URL, second form d.addCallback(self.log, "_get_from_uri2") def _get_from_uri2(res): - return self.GET("uri?uri=%s" % (self.uri,)) + return self.GET("uri?uri=%s" % (str(self.uri, "utf-8"),)) d.addCallback(_get_from_uri2) d.addCallback(_got_from_uri) @@ -2006,9 +2005,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): @defer.inlineCallbacks def _get_from_bogus_uri(res): d1 = self.GET("uri/%s?filename=%s" - % (self.mangle_uri(self.uri), "mydata567")) + % (str(self.mangle_uri(self.uri), "utf-8"), "mydata567")) e = yield self.assertFailure(d1, Error) - self.assertEquals(e.status, "410") + self.assertEquals(e.status, b"410") d.addCallback(_get_from_bogus_uri) d.addCallback(self.log, "_got_from_bogus_uri", level=log.UNUSUAL) @@ -2092,14 +2091,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): workdir = os.path.join(self.getdir("client0"), "helper") incfile = os.path.join(workdir, "CHK_incoming", "spurious") f = open(incfile, "wb") - f.write("small file") + f.write(b"small file") f.close() then = time.time() - 86400*3 now = time.time() os.utime(incfile, (now, then)) encfile = os.path.join(workdir, "CHK_encoding", "spurious") f = open(encfile, "wb") - f.write("less small file") + f.write(b"less small file") f.close() os.utime(encfile, (now, then)) d.addCallback(_got_helper_status) @@ -2140,7 +2139,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(lambda res: self.GET("statistics")) def _got_stats(res): self.failUnlessIn("Operational Statistics", res) - self.failUnlessIn(" 'downloader.files_downloaded': 5,", res) + self.failUnlessIn(' "downloader.files_downloaded": 5,', res) d.addCallback(_got_stats) d.addCallback(lambda res: self.GET("statistics?t=json")) def _got_stats_json(res): @@ -2348,7 +2347,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): for i in range(10): fn = os.path.join(self.basedir, "file%d" % i) files.append(fn) - data = "data to be uploaded: file%d\n" % i + data = b"data to be uploaded: file%d\n" % i datas.append(data) with open(fn, "wb") as f: f.write(data) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index cebe709c1..2d9ac2b28 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1760,7 +1760,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_PUT_NEWFILEURL_unlinked_bad_format(self): contents = self.NEWFILE_CONTENTS * 300000 yield self.assertHTTPError(self.webish_url + "/uri?format=foo", 400, - "Unknown format:", + "Unknown format: foo", method="put", data=contents) def test_PUT_NEWFILEURL_range_bad(self): @@ -1813,7 +1813,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_PUT_NEWFILEURL_bad_t(self): d = self.shouldFail2(error.Error, "PUT_bad_t", "400 Bad Request", - "PUT to a file: bad t=", + "PUT to a file: bad t=bogus", self.PUT, self.public_url + "/foo/bar.txt?t=bogus", b"contents") return d @@ -2344,7 +2344,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_PUT_NEWDIRURL_bad_format(self): url = (self.webish_url + self.public_url + "/foo/newdir=?t=mkdir&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="put", data="") def test_POST_NEWDIRURL(self): @@ -2377,7 +2377,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_NEWDIRURL_bad_format(self): url = (self.webish_url + self.public_url + "/foo/newdir?t=mkdir&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post", data="") def test_POST_NEWDIRURL_emptyname(self): @@ -2454,7 +2454,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi (newkids, caps) = self._create_initial_children() url = (self.webish_url + self.public_url + "/foo/newdir?t=mkdir-with-children&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post", data=json.dumps(newkids).encode("utf-8")) def test_POST_NEWDIRURL_immutable(self): @@ -2578,7 +2578,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_PUT_NEWDIRURL_mkdirs_bad_format(self): url = (self.webish_url + self.public_url + "/foo/subdir/newdir?t=mkdir&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="put", data="") def test_DELETE_DIRURL(self): @@ -2857,7 +2857,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi url = self.webish_url + "/uri?t=upload&format=foo" body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) yield self.assertHTTPError(url, 400, - "Unknown format:", + "Unknown format: foo", method="post", data=body, headers=headers) def test_POST_upload_format(self): @@ -2892,7 +2892,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_upload_bad_format(self): url = self.webish_url + self.public_url + "/foo?t=upload&format=foo" body, headers = self.build_form(file=("foo.txt", self.NEWFILE_CONTENTS * 300000)) - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post", data=body, headers=headers) def test_POST_upload_mutable(self): @@ -3388,7 +3388,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_POST_mkdir_bad_format(self): url = (self.webish_url + self.public_url + "/foo?t=mkdir&name=newdir&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post") def test_POST_mkdir_initial_children(self): @@ -3440,7 +3440,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi (newkids, caps) = self._create_initial_children() url = (self.webish_url + self.public_url + "/foo?t=mkdir-with-children&name=newdir&format=foo") - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post", data=json.dumps(newkids).encode("utf-8")) def test_POST_mkdir_immutable(self): @@ -3519,7 +3519,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_POST_mkdir_no_parentdir_noredirect_bad_format(self): url = self.webish_url + self.public_url + "/uri?t=mkdir&format=foo" - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="post") def test_POST_mkdir_no_parentdir_noredirect2(self): @@ -4462,7 +4462,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_PUT_NEWFILEURL_bad_format(self): new_contents = self.NEWFILE_CONTENTS * 300000 url = self.webish_url + self.public_url + "/foo/foo.txt?format=foo" - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="put", data=new_contents) def test_PUT_NEWFILEURL_uri_replace(self): @@ -4595,7 +4595,7 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi @inlineCallbacks def test_PUT_mkdir_bad_format(self): url = self.webish_url + "/uri?t=mkdir&format=foo" - yield self.assertHTTPError(url, 400, "Unknown format:", + yield self.assertHTTPError(url, 400, "Unknown format: foo", method="put", data=b"") def test_POST_check(self): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f0cebbba..24efc4add 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -180,9 +180,9 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_storage_client", "allmydata.test.test_storage_web", - # Only partially ported, test_filesystem_with_cli_in_subprocess and - # test_filesystem methods aren't ported yet, should be done once CLI and - # web are ported respectively. + # Only partially ported, test_filesystem_with_cli_in_subprocess isn't + # ported yet, nor is part of test_filesystem (the call to _test_cli). This + # should be done once CLI is ported. "allmydata.test.test_system", "allmydata.test.test_time_format", diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py index 5309c2612..e25fc011d 100644 --- a/src/allmydata/web/common.py +++ b/src/allmydata/web/common.py @@ -148,7 +148,7 @@ def get_format(req, default="CHK"): elif arg.upper() == b"MDMF": return "MDMF" else: - raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % arg, + raise WebError("Unknown format: %s, I know CHK, SDMF, MDMF" % ensure_str(arg), http.BAD_REQUEST) def get_mutable_type(file_format): # accepts result of get_format() diff --git a/src/allmydata/web/filenode.py b/src/allmydata/web/filenode.py index 4345d4b4a..97e88a8a3 100644 --- a/src/allmydata/web/filenode.py +++ b/src/allmydata/web/filenode.py @@ -1,4 +1,5 @@ from past.builtins import unicode, long +from six import ensure_str from twisted.web import http, static from twisted.internet import defer @@ -130,7 +131,7 @@ class PlaceHolderNodeHandler(Resource, ReplaceMeMixin): if t == b"uri": return self.replace_me_with_a_childcap(req, self.client, replace) - raise WebError("PUT to a file: bad t=%s" % t) + raise WebError("PUT to a file: bad t=%s" % ensure_str(t)) @render_exception def render_POST(self, req): @@ -287,7 +288,7 @@ class FileNodeHandler(Resource, ReplaceMeMixin, object): assert self.parentnode and self.name return self.replace_me_with_a_childcap(req, self.client, replace) - raise WebError("PUT to a file: bad t=%s" % t) + raise WebError("PUT to a file: bad t=%s" % ensure_str(t)) @render_exception def render_POST(self, req): diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 8f78d8a0b..29533523b 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -152,7 +152,7 @@ class ReloadMixin(object): @renderer def refresh(self, req, tag): if self.monitor.is_finished(): - return b"" + return "" tag.attributes["http-equiv"] = "refresh" tag.attributes["content"] = str(self.REFRESH_TIME) return tag diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3a717567e..82dc411d4 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1,6 +1,5 @@ from past.builtins import long, unicode -import pprint import itertools import hashlib from twisted.internet import defer @@ -61,7 +60,7 @@ class UploadResultsRendererMixin(Element): return "None" ul = tags.ul() for shnum, servers in sorted(sharemap.items()): - server_names = ', '.join([s.get_name() for s in servers]) + server_names = ', '.join([unicode(s.get_name(), "utf-8") for s in servers]) ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) return ul d.addCallback(_render) @@ -75,7 +74,7 @@ class UploadResultsRendererMixin(Element): if servermap is None: return "None" ul = tags.ul() - for server, shnums in sorted(servermap.items()): + for server, shnums in sorted(servermap.items(), key=id): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) ul(tags.li("[%s] got share%s: %s" % (server.get_name(), plural(shnums), shares_s))) @@ -1595,5 +1594,5 @@ class StatisticsElement(Element): @renderer def raw(self, req, tag): - raw = pprint.pformat(self._stats) + raw = json.dumps(self._stats, sort_keys=True, indent=4) return tag(raw)