diff --git a/newsfragments/3618.minor b/newsfragments/3618.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index d377b6396..8b8619e47 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -30,5 +30,5 @@ def remove_prefix(s_bytes, prefix): if s_bytes.startswith(prefix): return s_bytes[len(prefix):] raise BadPrefixError( - "did not see expected '{}' prefix".format(prefix) + "did not see expected '{!r}' prefix".format(prefix) ) diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index d8154f16e..17467459b 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -164,8 +164,10 @@ class CompleteBinaryTreeMixin(object): def dump(self): lines = [] for i,depth in self.depth_first(): - lines.append("%s%3d: %s" % (" "*depth, i, - base32.b2a_or_none(self[i]))) + value = base32.b2a_or_none(self[i]) + if value is not None: + value = str(value, "utf-8") + lines.append("%s%3d: %s" % (" "*depth, i, value)) return "\n".join(lines) + "\n" def get_leaf_index(self, leafnum): diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index c1eb5480d..30abc68c6 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -67,12 +67,12 @@ class ValidatedExtendedURIProxy(object): self.crypttext_hash = None def __str__(self): - return "<%s %s>" % (self.__class__.__name__, self._verifycap.to_string()) + return "<%s %r>" % (self.__class__.__name__, self._verifycap.to_string()) def _check_integrity(self, data): h = uri_extension_hash(data) if h != self._verifycap.uri_extension_hash: - msg = ("The copy of uri_extension we received from %s was bad: wanted %s, got %s" % + msg = ("The copy of uri_extension we received from %s was bad: wanted %r, got %r" % (self._readbucketproxy, base32.b2a(self._verifycap.uri_extension_hash), base32.b2a(h))) @@ -234,7 +234,7 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin): UEB""" precondition(share_hash_tree[0] is not None, share_hash_tree) prefix = "%d-%s-%s" % (sharenum, bucket, - base32.b2a(share_hash_tree[0][:8])[:12]) + str(base32.b2a(share_hash_tree[0][:8])[:12], "ascii")) log.PrefixingLogMixin.__init__(self, facility="tahoe.immutable.download", prefix=prefix) @@ -427,7 +427,7 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin): received from the remote peer were bad.""") self.log(" have candidate_share_hash: %s" % bool(candidate_share_hash)) self.log(" block length: %d" % len(blockdata)) - self.log(" block hash: %s" % base32.b2a_or_none(blockhash)) + self.log(" block hash: %r" % base32.b2a_or_none(blockhash)) if len(blockdata) < 100: self.log(" block data: %r" % (blockdata,)) else: diff --git a/src/allmydata/immutable/downloader/fetcher.py b/src/allmydata/immutable/downloader/fetcher.py index 6570346de..4e8b7f926 100644 --- a/src/allmydata/immutable/downloader/fetcher.py +++ b/src/allmydata/immutable/downloader/fetcher.py @@ -127,7 +127,7 @@ class SegmentFetcher(object): # we could have sent something if we'd been allowed to pull # more shares per server. Increase the limit and try again. self._max_shares_per_server += 1 - log.msg("SegmentFetcher(%s) increasing diversity limit to %d" + log.msg("SegmentFetcher(%r) increasing diversity limit to %d" % (self._node._si_prefix, self._max_shares_per_server), level=log.NOISY, umid="xY2pBA") # Also ask for more shares, in the hopes of achieving better diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index c01fa22cc..10ce0e5c7 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -500,7 +500,7 @@ class DownloadNode(object): return (offset, segment, decodetime) except (BadHashError, NotEnoughHashesError): format = ("hash failure in ciphertext_hash_tree:" - " segnum=%(segnum)d, SI=%(si)s") + " segnum=%(segnum)d, SI=%(si)r") log.msg(format=format, segnum=segnum, si=self._si_prefix, failure=Failure(), level=log.WEIRD, parent=self._lp, umid="MTwNnw") diff --git a/src/allmydata/immutable/downloader/segmentation.py b/src/allmydata/immutable/downloader/segmentation.py index b1d42dcc4..6d7cb7676 100644 --- a/src/allmydata/immutable/downloader/segmentation.py +++ b/src/allmydata/immutable/downloader/segmentation.py @@ -120,7 +120,7 @@ class Segmentation(object): # we didn't get the first byte, so we can't use this segment log.msg("Segmentation handed wrong data:" " want [%d-%d), given [%d-%d), for segnum=%d," - " for si=%s" + " for si=%r" % (self._offset, self._offset+self._size, segment_start, segment_start+len(segment), wanted_segnum, self._node._si_prefix), diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index f279018ba..86b99e99c 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -108,7 +108,7 @@ class Share(object): self.had_corruption = False # for unit tests def __repr__(self): - return "Share(sh%d-on-%s)" % (self._shnum, self._server.get_name()) + return "Share(sh%d-on-%s)" % (self._shnum, str(self._server.get_name(), "utf-8")) def is_alive(self): # XXX: reconsider. If the share sees a single error, should it remain diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6736e4e06..3db9f096e 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -175,7 +175,7 @@ class WriteBucketProxy(object): self._offset_data = offset_data def __repr__(self): - return "" % self._server.get_name() + return "" % self._server.get_name() def put_header(self): return self._write(0, self._offset_data) @@ -317,7 +317,7 @@ class ReadBucketProxy(object): return self._server.get_serverid() def __repr__(self): - return "" % \ + return "" % \ (id(self), self._server.get_name(), si_b2a(self._storage_index)) def _start_if_needed(self): diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 2d2c5c1f5..2c4b9db78 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -81,7 +81,7 @@ class CHKCheckerAndUEBFetcher(object): def _got_response(self, buckets, server): # buckets is a dict: maps shum to an rref of the server who holds it shnums_s = ",".join([str(shnum) for shnum in buckets]) - self.log("got_response: [%s] has %d shares (%s)" % + self.log("got_response: [%r] has %d shares (%s)" % (server.get_name(), len(buckets), shnums_s), level=log.NOISY) self._found_shares.update(buckets.keys()) @@ -167,7 +167,7 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warn self._upload_status.set_storage_index(storage_index) self._upload_status.set_status("fetching ciphertext") self._upload_status.set_progress(0, 1.0) - self._helper.log("CHKUploadHelper starting for SI %s" % self._upload_id, + self._helper.log("CHKUploadHelper starting for SI %r" % self._upload_id, parent=log_number) self._storage_broker = storage_broker diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index b2c4c744d..3d794abf1 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -278,7 +278,7 @@ class ServerTracker(object): self.cancel_secret = bucket_cancel_secret def __repr__(self): - return ("" + return ("" % (self._server.get_name(), si_b2a(self.storage_index)[:5])) def get_server(self): @@ -338,7 +338,7 @@ class ServerTracker(object): def str_shareloc(shnum, bucketwriter): - return "%s: %s" % (shnum, bucketwriter.get_servername(),) + return "%s: %s" % (shnum, ensure_str(bucketwriter.get_servername()),) @implementer(IPeerSelector) @@ -590,7 +590,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): d = timeout_call(self._reactor, tracker.ask_about_existing_shares(), 15) d.addBoth(self._handle_existing_response, tracker) ds.append(d) - self.log("asking server %s for any existing shares" % + self.log("asking server %r for any existing shares" % (tracker.get_name(),), level=log.NOISY) for tracker in write_trackers: @@ -605,7 +605,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): d.addErrback(timed_out, tracker) d.addBoth(self._handle_existing_write_response, tracker, set()) ds.append(d) - self.log("asking server %s for any existing shares" % + self.log("asking server %r for any existing shares" % (tracker.get_name(),), level=log.NOISY) trackers = set(write_trackers) | set(readonly_trackers) @@ -749,7 +749,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): buckets = res if buckets: self.serverids_with_shares.add(serverid) - self.log("response to get_buckets() from server %s: alreadygot=%s" + self.log("response to get_buckets() from server %r: alreadygot=%s" % (tracker.get_name(), tuple(sorted(buckets))), level=log.NOISY) for bucket in buckets: @@ -818,7 +818,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): self.homeless_shares.remove(shnum) if self._status: - self._status.set_status("Contacting Servers [%s] (first query)," + self._status.set_status("Contacting Servers [%r] (first query)," " %d shares left.." % (tracker.get_name(), len(self.homeless_shares))) @@ -845,7 +845,7 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): else: (alreadygot, allocated) = res - self.log("response to allocate_buckets() from server %s: alreadygot=%s, allocated=%s" + self.log("response to allocate_buckets() from server %r: alreadygot=%s, allocated=%s" % (tracker.get_name(), tuple(sorted(alreadygot)), tuple(sorted(allocated))), level=log.NOISY) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 339c5a0ac..dcc2fd2c0 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -300,7 +300,7 @@ class IntroducerService(service.MultiService, Referenceable): level=log.UNUSUAL, umid="jfGMXQ") def remote_subscribe_v2(self, subscriber, service_name, subscriber_info): - self.log("introducer: subscription[%s] request at %s" + self.log("introducer: subscription[%r] request at %r" % (service_name, subscriber), umid="U3uzLg") service_name = ensure_text(service_name) subscriber_info = dictutil.UnicodeKeyDict({ diff --git a/src/allmydata/mutable/checker.py b/src/allmydata/mutable/checker.py index e3e5951f0..0899168c3 100644 --- a/src/allmydata/mutable/checker.py +++ b/src/allmydata/mutable/checker.py @@ -9,6 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from six import ensure_str from allmydata.uri import from_string from allmydata.util import base32, log, dictutil @@ -202,7 +203,7 @@ class MutableChecker(object): serverid = server.get_serverid() locator = (server, self._storage_index, shnum) corrupt_share_locators.append(locator) - s = "%s-sh%d" % (server.get_name(), shnum) + s = "%s-sh%d" % (ensure_str(server.get_name()), shnum) if f.check(CorruptShareError): ft = f.value.reason else: diff --git a/src/allmydata/mutable/common.py b/src/allmydata/mutable/common.py index 802681ae7..87951c7b2 100644 --- a/src/allmydata/mutable/common.py +++ b/src/allmydata/mutable/common.py @@ -63,7 +63,7 @@ class CorruptShareError(BadShareError): self.shnum = shnum self.reason = reason def __str__(self): - return "" % (self.__class__.__name__, id(self), self.is_readonly() and 'RO' or 'RW', self._uri.abbrev()) + return "<%s %x %s %r>" % (self.__class__.__name__, id(self), self.is_readonly() and 'RO' or 'RW', self._uri.abbrev()) else: return "<%s %x %s %s>" % (self.__class__.__name__, id(self), None, None) diff --git a/src/allmydata/mutable/publish.py b/src/allmydata/mutable/publish.py index 86ceb022c..a7bca6cba 100644 --- a/src/allmydata/mutable/publish.py +++ b/src/allmydata/mutable/publish.py @@ -915,7 +915,7 @@ class Publish(object): def log_goal(self, goal, message=""): logmsg = [message] for (shnum, server) in sorted([(s,p) for (p,s) in goal], key=lambda t: (id(t[0]), id(t[1]))): - logmsg.append("sh%d to [%s]" % (shnum, server.get_name())) + logmsg.append("sh%d to [%r]" % (shnum, server.get_name())) self.log("current goal: %s" % (", ".join(logmsg)), level=log.NOISY) self.log("we are planning to push new seqnum=#%d" % self._new_seqnum, level=log.NOISY) @@ -999,7 +999,7 @@ class Publish(object): return server = writer.server - lp = self.log("_got_write_answer from %s, share %d" % + lp = self.log("_got_write_answer from %r, share %d" % (server.get_name(), writer.shnum)) now = time.time() @@ -1135,14 +1135,14 @@ class Publish(object): (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = expected_version msg = ("somebody modified the share on us:" - " shnum=%d: I thought they had #%d:R=%s," % + " shnum=%d: I thought they had #%d:R=%r," % (shnum, seqnum, base32.b2a(root_hash)[:4])) if unknown_format: msg += (" but I don't know how to read share" " format %d" % version) else: - msg += " but testv reported #%d:R=%s" % \ + msg += " but testv reported #%d:R=%r" % \ (other_seqnum, base32.b2a(other_roothash)[:4]) self.log(msg, parent=lp, level=log.NOISY) # if expected_version==None, then we didn't expect to see a diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index c6d4e2e40..32aaa72e5 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -574,7 +574,7 @@ class Retrieve(object): remote server (with no guarantee of success) that its share is corrupt. """ - self.log("marking share %d on server %s as bad" % \ + self.log("marking share %d on server %r as bad" % \ (shnum, server.get_name())) prefix = self.verinfo[-2] self.servermap.mark_bad_share(server, shnum, prefix) diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index be8033f4d..d793d1dcd 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -11,6 +11,7 @@ if PY2: # Doesn't import str to prevent API leakage on Python 2 from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 from past.builtins import unicode +from six import ensure_str import sys, time, copy from zope.interface import implementer @@ -202,8 +203,8 @@ class ServerMap(object): (seqnum, root_hash, IV, segsize, datalength, k, N, prefix, offsets_tuple) = verinfo print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" % - (server.get_name(), shnum, - seqnum, base32.b2a(root_hash)[:4], k, N, + (unicode(server.get_name(), "utf-8"), shnum, + seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N, datalength), file=out) if self._problems: print("%d PROBLEMS" % len(self._problems), file=out) @@ -869,7 +870,7 @@ class ServermapUpdater(object): # versions. self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), - server.get_name(), shnum, + ensure_str(server.get_name()), shnum, k, n, segsize, datalen), parent=lp) self._valid_versions.add(verinfo) @@ -943,13 +944,13 @@ class ServermapUpdater(object): alleged_privkey_s = self._node._decrypt_privkey(enc_privkey) alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) if alleged_writekey != self._node.get_writekey(): - self.log("invalid privkey from %s shnum %d" % + self.log("invalid privkey from %r shnum %d" % (server.get_name(), shnum), parent=lp, level=log.WEIRD, umid="aJVccw") return # it's good - self.log("got valid privkey from shnum %d on serverid %s" % + self.log("got valid privkey from shnum %d on serverid %r" % (shnum, server.get_name()), parent=lp) privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) @@ -1211,9 +1212,9 @@ class ServermapUpdater(object): break more_queries.append(self.extra_servers.pop(0)) - self.log(format="sending %(more)d more queries: %(who)s", + self.log(format="sending %(more)d more queries: %(who)d", more=len(more_queries), - who=" ".join(["[%s]" % s.get_name() for s in more_queries]), + who=" ".join(["[%r]" % s.get_name() for s in more_queries]), level=log.NOISY) for server in more_queries: diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index e4bcf165e..b8aeee91e 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -6,6 +6,7 @@ except ImportError: pass from future.utils import bchr +from past.builtins import unicode # do not import any allmydata modules at this level. Do that from inside # individual functions instead. @@ -90,27 +91,34 @@ def dump_immutable_chk_share(f, out, options): "crypttext_hash", "crypttext_root_hash", "share_root_hash", "UEB_hash") display_keys = {"size": "file_size"} + + def to_string(v): + if isinstance(v, bytes): + return unicode(v, "utf-8") + else: + return str(v) + for k in keys1: if k in unpacked: dk = display_keys.get(k, k) - print("%20s: %s" % (dk, unpacked[k]), file=out) + print("%20s: %s" % (dk, to_string(unpacked[k])), file=out) print(file=out) for k in keys2: if k in unpacked: dk = display_keys.get(k, k) - print("%20s: %s" % (dk, unpacked[k]), file=out) + print("%20s: %s" % (dk, to_string(unpacked[k])), file=out) print(file=out) for k in keys3: if k in unpacked: dk = display_keys.get(k, k) - print("%20s: %s" % (dk, unpacked[k]), file=out) + print("%20s: %s" % (dk, to_string(unpacked[k])), file=out) leftover = set(unpacked.keys()) - set(keys1 + keys2 + keys3) if leftover: print(file=out) print("LEFTOVER:", file=out) for k in sorted(leftover): - print("%20s: %s" % (k, unpacked[k]), file=out) + print("%20s: %s" % (k, to_string(unpacked[k])), file=out) # the storage index isn't stored in the share itself, so we depend upon # knowing the parent directory name to get it @@ -197,7 +205,7 @@ def dump_mutable_share(options): print(file=out) print("Mutable slot found:", file=out) print(" share_type: %s" % share_type, file=out) - print(" write_enabler: %s" % base32.b2a(WE), file=out) + print(" write_enabler: %s" % unicode(base32.b2a(WE), "utf-8"), file=out) print(" WE for nodeid: %s" % idlib.nodeid_b2a(nodeid), file=out) print(" num_extra_leases: %d" % num_extra_leases, file=out) print(" container_size: %d" % container_size, file=out) @@ -209,8 +217,8 @@ def dump_mutable_share(options): print(" ownerid: %d" % lease.owner_num, file=out) when = format_expiration_time(lease.expiration_time) print(" expires in %s" % when, file=out) - print(" renew_secret: %s" % base32.b2a(lease.renew_secret), file=out) - print(" cancel_secret: %s" % base32.b2a(lease.cancel_secret), file=out) + print(" renew_secret: %s" % unicode(base32.b2a(lease.renew_secret), "utf-8"), file=out) + print(" cancel_secret: %s" % unicode(base32.b2a(lease.cancel_secret), "utf-8"), file=out) print(" secrets are for nodeid: %s" % idlib.nodeid_b2a(lease.nodeid), file=out) else: print("No leases.", file=out) @@ -258,8 +266,8 @@ def dump_SDMF_share(m, length, options): print(" SDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % base32.b2a(root_hash), file=out) - print(" IV: %s" % base32.b2a(IV), file=out) + print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) + print(" IV: %s" % unicode(base32.b2a(IV), "utf-8"), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) print(" segsize: %d" % segsize, file=out) @@ -352,7 +360,7 @@ def dump_MDMF_share(m, length, options): print(" MDMF contents:", file=out) print(" seqnum: %d" % seqnum, file=out) - print(" root_hash: %s" % base32.b2a(root_hash), file=out) + print(" root_hash: %s" % unicode(base32.b2a(root_hash), "utf-8"), file=out) #print(" IV: %s" % base32.b2a(IV), file=out) print(" required_shares: %d" % k, file=out) print(" total_shares: %d" % N, file=out) @@ -745,7 +753,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): print("SDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, base32.b2a(root_hash), + seqnum, unicode(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) elif share_type == "MDMF": from allmydata.mutable.layout import MDMFSlotReadProxy @@ -774,7 +782,7 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): offsets) = verinfo print("MDMF %s %d/%d %d #%d:%s %d %s" % \ (si_s, k, N, datalen, - seqnum, base32.b2a(root_hash), + seqnum, unicode(base32.b2a(root_hash), "utf-8"), expiration, quote_output(abs_sharefile)), file=out) else: print("UNKNOWN mutable %s" % quote_output(abs_sharefile), file=out) @@ -808,8 +816,8 @@ def describe_share(abs_sharefile, si_s, shnum_s, now, out): ueb_hash = unpacked["UEB_hash"] print("CHK %s %d/%d %d %s %d %s" % (si_s, k, N, filesize, - ueb_hash, expiration, - quote_output(abs_sharefile)), file=out) + unicode(ueb_hash, "utf-8"), expiration, + quote_output(abs_sharefile)), file=out) else: print("UNKNOWN really-unknown %s" % quote_output(abs_sharefile), file=out) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index eb1572dcb..75c554562 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -38,7 +38,6 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - import re, time, hashlib # On Python 2 this will be the backport. @@ -820,7 +819,7 @@ class NativeStorageServer(service.MultiService): return self def __repr__(self): - return "" % self.get_name() + return "" % self.get_name() def get_serverid(self): return self._server_id def get_version(self): @@ -844,10 +843,10 @@ class NativeStorageServer(service.MultiService): version = self.get_version() if version is None: return None - protocol_v1_version = version.get('http://allmydata.org/tahoe/protocols/storage/v1', UnicodeKeyDict()) - available_space = protocol_v1_version.get('available-space') + protocol_v1_version = version.get(b'http://allmydata.org/tahoe/protocols/storage/v1', BytesKeyDict()) + available_space = protocol_v1_version.get(b'available-space') if available_space is None: - available_space = protocol_v1_version.get('maximum-immutable-share-size', None) + available_space = protocol_v1_version.get(b'maximum-immutable-share-size', None) return available_space def start_connecting(self, trigger_cb): diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 19c046eca..e3ac48290 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -14,13 +14,23 @@ Rather than defining interesting APIs for other code to use, this just causes some side-effects which make things better when the test suite runs. """ +from future.utils import PY3 + +import warnings from traceback import extract_stack, format_list + from foolscap.pb import Listener from twisted.python.log import err from twisted.application import service - from foolscap.logging.incident import IncidentQualifier + +if PY3: + # Error on BytesWarnings, to catch things like str(b""), but only for + # allmydata code. + warnings.filterwarnings("error", category=BytesWarning, module="allmydata.*") + + class NonQualifier(IncidentQualifier, object): def check_event(self, ev): return False diff --git a/src/allmydata/test/mutable/test_update.py b/src/allmydata/test/mutable/test_update.py index 65af06486..da5d53e4c 100644 --- a/src/allmydata/test/mutable/test_update.py +++ b/src/allmydata/test/mutable/test_update.py @@ -114,9 +114,9 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): # with problems and display them separately gotmods = [mo.span() for mo in re.finditer(b'([A-Z]+)', got)] expmods = [mo.span() for mo in re.finditer(b'([A-Z]+)', expected)] - gotspans = ["%d:%d=%s" % (start,end,got[start:end]) + gotspans = ["%d:%d=%r" % (start,end,got[start:end]) for (start,end) in gotmods] - expspans = ["%d:%d=%s" % (start,end,expected[start:end]) + expspans = ["%d:%d=%r" % (start,end,expected[start:end]) for (start,end) in expmods] #print("expecting: %s" % expspans) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index cbea0dfcd..2f75f9274 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -200,7 +200,8 @@ class NoNetworkServer(object): return self.serverid def get_name(self): - return idlib.shortnodeid_b2a(self.serverid) + # Other implementations return bytes. + return idlib.shortnodeid_b2a(self.serverid).encode("utf-8") def get_longname(self): return idlib.nodeid_b2a(self.serverid) def get_nickname(self): diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 8500d6bff..1a84f35ec 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -118,17 +118,17 @@ class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # ta class TestNativeStorageServer(unittest.TestCase): def test_get_available_space_new(self): nss = NativeStorageServerWithVersion( - { "http://allmydata.org/tahoe/protocols/storage/v1": - { "maximum-immutable-share-size": 111, - "available-space": 222, + { b"http://allmydata.org/tahoe/protocols/storage/v1": + { b"maximum-immutable-share-size": 111, + b"available-space": 222, } }) self.failUnlessEqual(nss.get_available_space(), 222) def test_get_available_space_old(self): nss = NativeStorageServerWithVersion( - { "http://allmydata.org/tahoe/protocols/storage/v1": - { "maximum-immutable-share-size": 111, + { b"http://allmydata.org/tahoe/protocols/storage/v1": + { b"maximum-immutable-share-size": 111, } }) self.failUnlessEqual(nss.get_available_space(), 111) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 8214c312c..ce575ce7a 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1072,7 +1072,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(_do_upload) def _upload_done(results): theuri = results.get_uri() - log.msg("upload finished: uri is %s" % (theuri,)) + log.msg("upload finished: uri is %r" % (theuri,)) self.uri = theuri assert isinstance(self.uri, bytes), self.uri self.cap = uri.from_string(self.uri) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index a659dac38..70742b7b2 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -99,7 +99,7 @@ class CHKFileURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)), int(mo.group(3)), int(mo.group(4)), int(mo.group(5))) @@ -290,7 +290,7 @@ class ReadonlySSKFileURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): @@ -300,7 +300,7 @@ class ReadonlySSKFileURI(_BaseURI): base32.b2a(self.fingerprint)) def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.abbrev()) + return "<%s %r>" % (self.__class__.__name__, self.abbrev()) def abbrev(self): return base32.b2a(self.readkey[:5]) @@ -336,7 +336,7 @@ class SSKVerifierURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): @@ -375,7 +375,7 @@ class WriteableMDMFFileURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): @@ -386,7 +386,7 @@ class WriteableMDMFFileURI(_BaseURI): return ret def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.abbrev()) + return "<%s %r>" % (self.__class__.__name__, self.abbrev()) def abbrev(self): return base32.b2a(self.writekey[:5]) @@ -423,7 +423,7 @@ class ReadonlyMDMFFileURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) @@ -435,7 +435,7 @@ class ReadonlyMDMFFileURI(_BaseURI): return ret def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.abbrev()) + return "<%s %r>" % (self.__class__.__name__, self.abbrev()) def abbrev(self): return base32.b2a(self.readkey[:5]) @@ -471,7 +471,7 @@ class MDMFVerifierURI(_BaseURI): def init_from_string(cls, uri): mo = cls.STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): @@ -500,13 +500,13 @@ class _DirectoryBaseURI(_BaseURI): self._filenode_uri = filenode_uri def __repr__(self): - return "<%s %s>" % (self.__class__.__name__, self.abbrev()) + return "<%s %r>" % (self.__class__.__name__, self.abbrev()) @classmethod def init_from_string(cls, uri): mo = cls.BASE_STRING_RE.search(uri) if not mo: - raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) + raise BadURIError("%r doesn't look like a %s cap" % (uri, cls)) bits = uri[mo.end():] fn = cls.INNER_URI_CLASS.init_from_string( cls.INNER_URI_CLASS.BASE_STRING+bits) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 82dc411d4..6a8f689b3 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -76,7 +76,7 @@ class UploadResultsRendererMixin(Element): ul = tags.ul() for server, shnums in sorted(servermap.items(), key=id): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) - ul(tags.li("[%s] got share%s: %s" % (server.get_name(), + ul(tags.li("[%s] got share%s: %s" % (unicode(server.get_name(), "utf-8"), plural(shnums), shares_s))) return ul d.addCallback(_render) @@ -230,7 +230,9 @@ class UploadStatusElement(UploadResultsRendererMixin): si_s = base32.b2a_or_none(self._upload_status.get_storage_index()) if si_s is None: si_s = "(None)" - return tag(str(si_s)) + else: + si_s = unicode(si_s, "utf-8") + return tag(si_s) @renderer def helper(self, req, tag): @@ -920,7 +922,7 @@ class RetrieveStatusElement(Element): for server in sorted(per_server.keys(), key=lambda s: s.get_name()): times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) - l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Fetch Response Times: ", l) @@ -958,7 +960,9 @@ class PublishStatusElement(Element): si_s = base32.b2a_or_none(self._publish_status.get_storage_index()) if si_s is None: si_s = "(None)" - return tag(str(si_s)) + else: + si_s = unicode("utf-8") + return tag(si_s) @renderer def helper(self, req, tag): @@ -996,7 +1000,7 @@ class PublishStatusElement(Element): sharemap = servermap.make_sharemap() for shnum in sorted(sharemap.keys()): l(tags.li("%d -> Placed on " % shnum, - ", ".join(["[%s]" % server.get_name() + ", ".join(["[%s]" % unicode(server.get_name(), "utf-8") for server in sharemap[shnum]]))) return tag("Sharemap:", l) @@ -1078,7 +1082,7 @@ class PublishStatusElement(Element): for server in sorted(per_server.keys(), key=lambda s: s.get_name()): times_s = ", ".join([abbreviate_time(t) for t in per_server[server]]) - l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Response Times: ", l) @@ -1204,7 +1208,7 @@ class MapupdateStatusElement(Element): else: times.append("privkey(" + abbreviate_time(t) + ")") times_s = ", ".join(times) - l(tags.li("[%s]: %s" % (server.get_name(), times_s))) + l(tags.li("[%s]: %s" % (unicode(server.get_name(), "utf-8"), times_s))) return tags.li("Per-Server Response Times: ", l) diff --git a/tox.ini b/tox.ini index 77f5faede..bc18ed519 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ deps = # more useful results. usedevelop = False # We use extras=test to get things like "mock" that are required for our unit -# tests. +# tests.8 extras = test setenv = @@ -62,20 +62,19 @@ commands = tahoe --version - !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}} + # Run tests with -b to catch bugs like `"%s" % (some_bytes,)`. -b makes + # Python emit BytesWarnings, and warnings configuration in + # src/allmydata/tests/__init__.py turns allmydata's BytesWarnings into + # exceptions. + !coverage: python -b -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}} # measuring coverage is somewhat slower than not measuring coverage # so only do it on request. - coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}} + coverage: python -b -m coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}} coverage: coverage combine coverage: coverage xml coverage: coverage report - # Also run tests with -bb to catch bugs like `"%s" % (some_bytes,)`. - # Eventually everything should run with this, but so far only fixed - # some of the Python3-ported modules. - python -bb -m twisted.trial --rterrors allmydata.test.web - [testenv:integration] setenv = COVERAGE_PROCESS_START=.coveragerc