From 498e69c72e01f198f38bf24f46cb08079cb21aed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 16 Sep 2020 14:57:55 -0400 Subject: [PATCH] Some progress. --- src/allmydata/immutable/checker.py | 20 +++++------ src/allmydata/test/test_encode.py | 56 ++++++++++++++++++------------ 2 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py index 36bf735dc..73d6a0263 100644 --- a/src/allmydata/immutable/checker.py +++ b/src/allmydata/immutable/checker.py @@ -117,7 +117,7 @@ class ValidatedExtendedURIProxy(object): # Next: things that are optional and not redundant: crypttext_hash - if d.has_key('crypttext_hash'): + if 'crypttext_hash' in d: self.crypttext_hash = d['crypttext_hash'] if len(self.crypttext_hash) != CRYPTO_VAL_SIZE: raise BadURIExtension('crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes' % (len(self.crypttext_hash),)) @@ -126,11 +126,11 @@ class ValidatedExtendedURIProxy(object): # Next: things that are optional, redundant, and required to be # consistent: codec_name, codec_params, tail_codec_params, # num_segments, size, needed_shares, total_shares - if d.has_key('codec_name'): + if 'codec_name' in d: if d['codec_name'] != "crs": raise UnsupportedErasureCodec(d['codec_name']) - if d.has_key('codec_params'): + if 'codec_params' in d: ucpss, ucpns, ucpts = codec.parse_params(d['codec_params']) if ucpss != self.segment_size: raise BadURIExtension("inconsistent erasure code params: " @@ -145,7 +145,7 @@ class ValidatedExtendedURIProxy(object): "self._verifycap.total_shares: %s" % (ucpts, self._verifycap.total_shares)) - if d.has_key('tail_codec_params'): + if 'tail_codec_params' in d: utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params']) if utcpss != self.tail_segment_size: raise BadURIExtension("inconsistent erasure code params: utcpss: %s != " @@ -162,7 +162,7 @@ class ValidatedExtendedURIProxy(object): "self._verifycap.total_shares: %s" % (utcpts, self._verifycap.total_shares)) - if d.has_key('num_segments'): + if 'num_segments' in d: if d['num_segments'] != self.num_segments: raise BadURIExtension("inconsistent num_segments: size: %s, " "segment_size: %s, computed_num_segments: %s, " @@ -170,18 +170,18 @@ class ValidatedExtendedURIProxy(object): self.segment_size, self.num_segments, d['num_segments'])) - if d.has_key('size'): + if 'size' in d: if d['size'] != self._verifycap.size: raise BadURIExtension("inconsistent size: URI size: %s, UEB size: %s" % (self._verifycap.size, d['size'])) - if d.has_key('needed_shares'): + if 'needed_shares' in d: if d['needed_shares'] != self._verifycap.needed_shares: raise BadURIExtension("inconsistent needed shares: URI needed shares: %s, UEB " "needed shares: %s" % (self._verifycap.total_shares, d['needed_shares'])) - if d.has_key('total_shares'): + if 'total_shares' in d: if d['total_shares'] != self._verifycap.total_shares: raise BadURIExtension("inconsistent total shares: URI total shares: %s, UEB " "total shares: %s" % (self._verifycap.total_shares, @@ -428,7 +428,7 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) self.log(" sharehashes:\n" + "\n".join(lines) + "\n") lines = [] - for i,h in blockhashes.items(): + for i,h in list(blockhashes.items()): lines.append("%3d: %s" % (i, base32.b2a_or_none(h))) log.msg(" blockhashes:\n" + "\n".join(lines) + "\n") raise BadOrMissingHash(le) @@ -695,7 +695,7 @@ class Checker(log.PrefixingLogMixin): bucketdict, success = result shareverds = [] - for (sharenum, bucket) in bucketdict.items(): + for (sharenum, bucket) in list(bucketdict.items()): d = self._download_and_verify(s, sharenum, bucket) shareverds.append(d) diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py index 68d62b90a..7cec5a7d3 100644 --- a/src/allmydata/test/test_encode.py +++ b/src/allmydata/test/test_encode.py @@ -1,3 +1,13 @@ +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import chr as byteschr, long + from zope.interface import implementer from twisted.trial import unittest from twisted.internet import defer @@ -15,7 +25,7 @@ class LostPeerError(Exception): pass def flip_bit(good): # flips the last bit - return good[:-1] + chr(ord(good[-1]) ^ 0x01) + return good[:-1] + byteschr(ord(good[-1]) ^ 0x01) @implementer(IStorageBucketWriter, IStorageBucketReader) class FakeBucketReaderWriterProxy(object): @@ -158,7 +168,7 @@ class FakeBucketReaderWriterProxy(object): def make_data(length): - data = "happy happy joy joy" * 100 + data = b"happy happy joy joy" * 100 assert length <= len(data) return data[:length] @@ -173,32 +183,32 @@ class ValidatedExtendedURIProxy(unittest.TestCase): if _TMP % K != 0: _TMP += (K - (_TMP % K)) TAIL_SEGSIZE = _TMP - _TMP = SIZE / SEGSIZE + _TMP = SIZE // SEGSIZE if SIZE % SEGSIZE != 0: _TMP += 1 NUM_SEGMENTS = _TMP mindict = { 'segment_size': SEGSIZE, - 'crypttext_root_hash': '0'*hashutil.CRYPTO_VAL_SIZE, - 'share_root_hash': '1'*hashutil.CRYPTO_VAL_SIZE } - optional_consistent = { 'crypttext_hash': '2'*hashutil.CRYPTO_VAL_SIZE, - 'codec_name': "crs", - 'codec_params': "%d-%d-%d" % (SEGSIZE, K, M), - 'tail_codec_params': "%d-%d-%d" % (TAIL_SEGSIZE, K, M), + 'crypttext_root_hash': b'0'*hashutil.CRYPTO_VAL_SIZE, + 'share_root_hash': b'1'*hashutil.CRYPTO_VAL_SIZE } + optional_consistent = { 'crypttext_hash': b'2'*hashutil.CRYPTO_VAL_SIZE, + 'codec_name': b"crs", + 'codec_params': b"%d-%d-%d" % (SEGSIZE, K, M), + 'tail_codec_params': b"%d-%d-%d" % (TAIL_SEGSIZE, K, M), 'num_segments': NUM_SEGMENTS, 'size': SIZE, 'needed_shares': K, 'total_shares': M, - 'plaintext_hash': "anything", - 'plaintext_root_hash': "anything", } + 'plaintext_hash': b"anything", + 'plaintext_root_hash': b"anything", } # optional_inconsistent = { 'crypttext_hash': ('2'*(hashutil.CRYPTO_VAL_SIZE-1), "", 77), optional_inconsistent = { 'crypttext_hash': (77,), - 'codec_name': ("digital fountain", ""), - 'codec_params': ("%d-%d-%d" % (SEGSIZE, K-1, M), - "%d-%d-%d" % (SEGSIZE-1, K, M), - "%d-%d-%d" % (SEGSIZE, K, M-1)), - 'tail_codec_params': ("%d-%d-%d" % (TAIL_SEGSIZE, K-1, M), - "%d-%d-%d" % (TAIL_SEGSIZE-1, K, M), - "%d-%d-%d" % (TAIL_SEGSIZE, K, M-1)), + 'codec_name': (b"digital fountain", b""), + 'codec_params': (b"%d-%d-%d" % (SEGSIZE, K-1, M), + b"%d-%d-%d" % (SEGSIZE-1, K, M), + b"%d-%d-%d" % (SEGSIZE, K, M-1)), + 'tail_codec_params': (b"%d-%d-%d" % (TAIL_SEGSIZE, K-1, M), + b"%d-%d-%d" % (TAIL_SEGSIZE-1, K, M), + b"%d-%d-%d" % (TAIL_SEGSIZE, K, M-1)), 'num_segments': (NUM_SEGMENTS-1,), 'size': (SIZE-1,), 'needed_shares': (K-1,), @@ -209,7 +219,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase): uebhash = hashutil.uri_extension_hash(uebstring) fb = FakeBucketReaderWriterProxy() fb.put_uri_extension(uebstring) - verifycap = uri.CHKFileVerifierURI(storage_index='x'*16, uri_extension_hash=uebhash, needed_shares=self.K, total_shares=self.M, size=self.SIZE) + verifycap = uri.CHKFileVerifierURI(storage_index=b'x'*16, uri_extension_hash=uebhash, needed_shares=self.K, total_shares=self.M, size=self.SIZE) vup = checker.ValidatedExtendedURIProxy(fb, verifycap) return vup.start() @@ -232,7 +242,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase): def test_reject_insufficient(self): dl = [] - for k in self.mindict.iterkeys(): + for k in self.mindict.keys(): insuffdict = self.mindict.copy() del insuffdict[k] d = self._test_reject(insuffdict) @@ -241,7 +251,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase): def test_accept_optional(self): dl = [] - for k in self.optional_consistent.iterkeys(): + for k in self.optional_consistent.keys(): mydict = self.mindict.copy() mydict[k] = self.optional_consistent[k] d = self._test_accept(mydict) @@ -250,7 +260,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase): def test_reject_optional(self): dl = [] - for k in self.optional_inconsistent.iterkeys(): + for k in self.optional_inconsistent.keys(): for v in self.optional_inconsistent[k]: mydict = self.mindict.copy() mydict[k] = v @@ -398,7 +408,7 @@ class Roundtrip(GridTestMixin, unittest.TestCase): self.basedir = self.mktemp() self.set_up_grid() self.c0 = self.g.clients[0] - DATA = "p"*size + DATA = b"p"*size d = self.upload(DATA) d.addCallback(lambda n: download_to_data(n)) def _downloaded(newdata):