diff --git a/src/allmydata/control.py b/src/allmydata/control.py index b4603b2ac..25ce3cea7 100644 --- a/src/allmydata/control.py +++ b/src/allmydata/control.py @@ -65,7 +65,7 @@ class ControlServer(Referenceable, service.Service): tempdir = tempfile.mkdtemp() filename = os.path.join(tempdir, "data") f = open(filename, "wb") - block = "a" * 8192 + block = b"a" * 8192 while size > 0: l = min(size, 8192) f.write(block[:l]) @@ -126,7 +126,7 @@ class ControlServer(Referenceable, service.Service): server_name = server.get_longname() storage_server = server.get_storage_server() start = time.time() - d = storage_server.get_buckets("\x00" * 16) + d = storage_server.get_buckets(b"\x00" * 16) def _done(ignored): stop = time.time() elapsed = stop - start @@ -138,7 +138,7 @@ class ControlServer(Referenceable, service.Service): d.addCallback(self._do_one_ping, everyone_left, results) def _average(res): averaged = {} - for server_name,times in results.iteritems(): + for server_name,times in results.items(): averaged[server_name] = sum(times) / len(times) return averaged d.addCallback(_average) @@ -168,19 +168,19 @@ class SpeedTest(object): fn = os.path.join(self.basedir, str(i)) if os.path.exists(fn): os.unlink(fn) - f = open(fn, "w") + f = open(fn, "wb") f.write(os.urandom(8)) s -= 8 while s > 0: chunk = min(s, 4096) - f.write("\x00" * chunk) + f.write(b"\x00" * chunk) s -= chunk f.close() def do_upload(self): d = defer.succeed(None) def _create_slot(res): - d1 = self.parent.create_mutable_file("") + d1 = self.parent.create_mutable_file(b"") def _created(n): self._n = n d1.addCallback(_created) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 0dd5ddc83..96d3e813c 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -2858,7 +2858,7 @@ class RIControlClient(RemoteInterface): @return: a dictionary mapping peerid to a float (RTT time in seconds) """ - return DictOf(str, float) + return DictOf(bytes, float) UploadResults = Any() #DictOf(bytes, bytes) diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index 550c37fde..e4bcf165e 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -645,7 +645,7 @@ def find_shares(options): from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path out = options.stdout - sharedir = storage_index_to_dir(si_a2b(options.si_s)) + sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8"))) for d in options.nodedirs: d = os.path.join(d, "storage", "shares", sharedir) if os.path.exists(d): diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index 91205a93c..7137ba28e 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -17,7 +17,7 @@ from twisted.application.internet import TimerService from zope.interface import implementer from foolscap.api import eventually -from allmydata.util import log +from allmydata.util import log, dictutil from allmydata.interfaces import IStatsProducer @implementer(IStatsProducer) @@ -79,15 +79,13 @@ class StatsProvider(service.MultiService): service.MultiService.__init__(self) self.node = node - self.counters = {} + self.counters = dictutil.UnicodeKeyDict() self.stats_producers = [] self.cpu_monitor = CPUUsageMonitor() self.cpu_monitor.setServiceParent(self) self.register_producer(self.cpu_monitor) def count(self, name, delta=1): - if isinstance(name, str): - name = name.encode("utf-8") val = self.counters.setdefault(name, 0) self.counters[name] = val + delta diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 235361cf8..8eac3837d 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1301,9 +1301,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): s = stats["stats"] self.failUnlessEqual(s["storage_server.accepting_immutable_shares"], 1) c = stats["counters"] - # Probably this should be Unicode eventually? But we haven't ported - # stats code yet. - self.failUnless(b"storage_server.allocate" in c) + self.failUnless("storage_server.allocate" in c) d.addCallback(_grab_stats) return d @@ -1608,7 +1606,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): # the key, which should cause the download to fail the post-download # plaintext_hash check. - @skipIf(PY3, "Python 3 web support hasn't happened yet.") def test_filesystem(self): self.basedir = "system/SystemTest/test_filesystem" self.data = LARGE_DATA @@ -1646,7 +1643,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "did _check_publish_private") d.addCallback(self._test_web) d.addCallback(self._test_control) - d.addCallback(self._test_cli) + if PY2: + # TODO when CLI is ported to Python 3, reenable. + d.addCallback(self._test_cli) # P now has four top-level children: # P/personal/sekrit data # P/s2-ro/ @@ -1900,9 +1899,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): if isinstance(value, tuple): filename, value = value form.append(b'Content-Disposition: form-data; name="%s"; ' - b'filename="%s"' % (name, filename.encode("utf-8"))) + b'filename="%s"' % (name.encode("utf-8"), filename.encode("utf-8"))) else: - form.append(b'Content-Disposition: form-data; name="%s"' % name) + form.append(b'Content-Disposition: form-data; name="%s"' % name.encode("utf-8")) form.append(b'') form.append(b"%s" % (value,)) form.append(sep) @@ -1959,22 +1958,22 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self.log, "done with _got_subdir1") d.addCallback(lambda res: self.GET(public + "/subdir1/mydata567")) def _got_data(page): - self.failUnlessEqual(page, self.data) + self.failUnlessEqual(page.encode("utf-8"), self.data) d.addCallback(_got_data) # download from a URI embedded in a URL d.addCallback(self.log, "_get_from_uri") def _get_from_uri(res): - return self.GET("uri/%s?filename=%s" % (self.uri, "mydata567")) + return self.GET("uri/%s?filename=%s" % (str(self.uri, "utf-8"), "mydata567")) d.addCallback(_get_from_uri) def _got_from_uri(page): - self.failUnlessEqual(page, self.data) + self.failUnlessEqual(page.encode("utf-8"), self.data) d.addCallback(_got_from_uri) # download from a URI embedded in a URL, second form d.addCallback(self.log, "_get_from_uri2") def _get_from_uri2(res): - return self.GET("uri?uri=%s" % (self.uri,)) + return self.GET("uri?uri=%s" % (str(self.uri, "utf-8"),)) d.addCallback(_get_from_uri2) d.addCallback(_got_from_uri) @@ -1983,9 +1982,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): @defer.inlineCallbacks def _get_from_bogus_uri(res): d1 = self.GET("uri/%s?filename=%s" - % (self.mangle_uri(self.uri), "mydata567")) + % (str(self.mangle_uri(self.uri), "utf-8"), "mydata567")) e = yield self.assertFailure(d1, Error) - self.assertEquals(e.status, "410") + self.assertEquals(e.status, b"410") d.addCallback(_get_from_bogus_uri) d.addCallback(self.log, "_got_from_bogus_uri", level=log.UNUSUAL) @@ -2069,14 +2068,14 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): workdir = os.path.join(self.getdir("client0"), "helper") incfile = os.path.join(workdir, "CHK_incoming", "spurious") f = open(incfile, "wb") - f.write("small file") + f.write(b"small file") f.close() then = time.time() - 86400*3 now = time.time() os.utime(incfile, (now, then)) encfile = os.path.join(workdir, "CHK_encoding", "spurious") f = open(encfile, "wb") - f.write("less small file") + f.write(b"less small file") f.close() os.utime(encfile, (now, then)) d.addCallback(_got_helper_status) @@ -2117,7 +2116,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(lambda res: self.GET("statistics")) def _got_stats(res): self.failUnlessIn("Operational Statistics", res) - self.failUnlessIn(" 'downloader.files_downloaded': 5,", res) + self.failUnlessIn(' "downloader.files_downloaded": 5,', res) d.addCallback(_got_stats) d.addCallback(lambda res: self.GET("statistics?t=json")) def _got_stats_json(res): @@ -2325,7 +2324,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): for i in range(10): fn = os.path.join(self.basedir, "file%d" % i) files.append(fn) - data = "data to be uploaded: file%d\n" % i + data = b"data to be uploaded: file%d\n" % i datas.append(data) with open(fn, "wb") as f: f.write(data) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 066928ca4..7098877f5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -179,9 +179,9 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_storage_client", "allmydata.test.test_storage_web", - # Only partially ported, test_filesystem_with_cli_in_subprocess and - # test_filesystem methods aren't ported yet, should be done once CLI and - # web are ported respectively. + # Only partially ported, test_filesystem_with_cli_in_subprocess isn't + # ported yet, nor is part of test_filesystem (the call to _test_cli). This + # should be done once CLI is ported. "allmydata.test.test_system", "allmydata.test.test_time_format", diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3a717567e..82dc411d4 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1,6 +1,5 @@ from past.builtins import long, unicode -import pprint import itertools import hashlib from twisted.internet import defer @@ -61,7 +60,7 @@ class UploadResultsRendererMixin(Element): return "None" ul = tags.ul() for shnum, servers in sorted(sharemap.items()): - server_names = ', '.join([s.get_name() for s in servers]) + server_names = ', '.join([unicode(s.get_name(), "utf-8") for s in servers]) ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) return ul d.addCallback(_render) @@ -75,7 +74,7 @@ class UploadResultsRendererMixin(Element): if servermap is None: return "None" ul = tags.ul() - for server, shnums in sorted(servermap.items()): + for server, shnums in sorted(servermap.items(), key=id): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) ul(tags.li("[%s] got share%s: %s" % (server.get_name(), plural(shnums), shares_s))) @@ -1595,5 +1594,5 @@ class StatisticsElement(Element): @renderer def raw(self, req, tag): - raw = pprint.pformat(self._stats) + raw = json.dumps(self._stats, sort_keys=True, indent=4) return tag(raw)