From 498e69c72e01f198f38bf24f46cb08079cb21aed Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 16 Sep 2020 14:57:55 -0400
Subject: [PATCH 1/6] Some progress.
---
src/allmydata/immutable/checker.py | 20 +++++------
src/allmydata/test/test_encode.py | 56 ++++++++++++++++++------------
2 files changed, 43 insertions(+), 33 deletions(-)
diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py
index 36bf735dc..73d6a0263 100644
--- a/src/allmydata/immutable/checker.py
+++ b/src/allmydata/immutable/checker.py
@@ -117,7 +117,7 @@ class ValidatedExtendedURIProxy(object):
# Next: things that are optional and not redundant: crypttext_hash
- if d.has_key('crypttext_hash'):
+ if 'crypttext_hash' in d:
self.crypttext_hash = d['crypttext_hash']
if len(self.crypttext_hash) != CRYPTO_VAL_SIZE:
raise BadURIExtension('crypttext_hash is required to be hashutil.CRYPTO_VAL_SIZE bytes, not %s bytes' % (len(self.crypttext_hash),))
@@ -126,11 +126,11 @@ class ValidatedExtendedURIProxy(object):
# Next: things that are optional, redundant, and required to be
# consistent: codec_name, codec_params, tail_codec_params,
# num_segments, size, needed_shares, total_shares
- if d.has_key('codec_name'):
+ if 'codec_name' in d:
if d['codec_name'] != "crs":
raise UnsupportedErasureCodec(d['codec_name'])
- if d.has_key('codec_params'):
+ if 'codec_params' in d:
ucpss, ucpns, ucpts = codec.parse_params(d['codec_params'])
if ucpss != self.segment_size:
raise BadURIExtension("inconsistent erasure code params: "
@@ -145,7 +145,7 @@ class ValidatedExtendedURIProxy(object):
"self._verifycap.total_shares: %s" %
(ucpts, self._verifycap.total_shares))
- if d.has_key('tail_codec_params'):
+ if 'tail_codec_params' in d:
utcpss, utcpns, utcpts = codec.parse_params(d['tail_codec_params'])
if utcpss != self.tail_segment_size:
raise BadURIExtension("inconsistent erasure code params: utcpss: %s != "
@@ -162,7 +162,7 @@ class ValidatedExtendedURIProxy(object):
"self._verifycap.total_shares: %s" % (utcpts,
self._verifycap.total_shares))
- if d.has_key('num_segments'):
+ if 'num_segments' in d:
if d['num_segments'] != self.num_segments:
raise BadURIExtension("inconsistent num_segments: size: %s, "
"segment_size: %s, computed_num_segments: %s, "
@@ -170,18 +170,18 @@ class ValidatedExtendedURIProxy(object):
self.segment_size,
self.num_segments, d['num_segments']))
- if d.has_key('size'):
+ if 'size' in d:
if d['size'] != self._verifycap.size:
raise BadURIExtension("inconsistent size: URI size: %s, UEB size: %s" %
(self._verifycap.size, d['size']))
- if d.has_key('needed_shares'):
+ if 'needed_shares' in d:
if d['needed_shares'] != self._verifycap.needed_shares:
raise BadURIExtension("inconsistent needed shares: URI needed shares: %s, UEB "
"needed shares: %s" % (self._verifycap.total_shares,
d['needed_shares']))
- if d.has_key('total_shares'):
+ if 'total_shares' in d:
if d['total_shares'] != self._verifycap.total_shares:
raise BadURIExtension("inconsistent total shares: URI total shares: %s, UEB "
"total shares: %s" % (self._verifycap.total_shares,
@@ -428,7 +428,7 @@ class ValidatedReadBucketProxy(log.PrefixingLogMixin):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
self.log(" sharehashes:\n" + "\n".join(lines) + "\n")
lines = []
- for i,h in blockhashes.items():
+ for i,h in list(blockhashes.items()):
lines.append("%3d: %s" % (i, base32.b2a_or_none(h)))
log.msg(" blockhashes:\n" + "\n".join(lines) + "\n")
raise BadOrMissingHash(le)
@@ -695,7 +695,7 @@ class Checker(log.PrefixingLogMixin):
bucketdict, success = result
shareverds = []
- for (sharenum, bucket) in bucketdict.items():
+ for (sharenum, bucket) in list(bucketdict.items()):
d = self._download_and_verify(s, sharenum, bucket)
shareverds.append(d)
diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py
index 68d62b90a..7cec5a7d3 100644
--- a/src/allmydata/test/test_encode.py
+++ b/src/allmydata/test/test_encode.py
@@ -1,3 +1,13 @@
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+ from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
+from past.builtins import chr as byteschr, long
+
from zope.interface import implementer
from twisted.trial import unittest
from twisted.internet import defer
@@ -15,7 +25,7 @@ class LostPeerError(Exception):
pass
def flip_bit(good): # flips the last bit
- return good[:-1] + chr(ord(good[-1]) ^ 0x01)
+ return good[:-1] + byteschr(ord(good[-1]) ^ 0x01)
@implementer(IStorageBucketWriter, IStorageBucketReader)
class FakeBucketReaderWriterProxy(object):
@@ -158,7 +168,7 @@ class FakeBucketReaderWriterProxy(object):
def make_data(length):
- data = "happy happy joy joy" * 100
+ data = b"happy happy joy joy" * 100
assert length <= len(data)
return data[:length]
@@ -173,32 +183,32 @@ class ValidatedExtendedURIProxy(unittest.TestCase):
if _TMP % K != 0:
_TMP += (K - (_TMP % K))
TAIL_SEGSIZE = _TMP
- _TMP = SIZE / SEGSIZE
+ _TMP = SIZE // SEGSIZE
if SIZE % SEGSIZE != 0:
_TMP += 1
NUM_SEGMENTS = _TMP
mindict = { 'segment_size': SEGSIZE,
- 'crypttext_root_hash': '0'*hashutil.CRYPTO_VAL_SIZE,
- 'share_root_hash': '1'*hashutil.CRYPTO_VAL_SIZE }
- optional_consistent = { 'crypttext_hash': '2'*hashutil.CRYPTO_VAL_SIZE,
- 'codec_name': "crs",
- 'codec_params': "%d-%d-%d" % (SEGSIZE, K, M),
- 'tail_codec_params': "%d-%d-%d" % (TAIL_SEGSIZE, K, M),
+ 'crypttext_root_hash': b'0'*hashutil.CRYPTO_VAL_SIZE,
+ 'share_root_hash': b'1'*hashutil.CRYPTO_VAL_SIZE }
+ optional_consistent = { 'crypttext_hash': b'2'*hashutil.CRYPTO_VAL_SIZE,
+ 'codec_name': b"crs",
+ 'codec_params': b"%d-%d-%d" % (SEGSIZE, K, M),
+ 'tail_codec_params': b"%d-%d-%d" % (TAIL_SEGSIZE, K, M),
'num_segments': NUM_SEGMENTS,
'size': SIZE,
'needed_shares': K,
'total_shares': M,
- 'plaintext_hash': "anything",
- 'plaintext_root_hash': "anything", }
+ 'plaintext_hash': b"anything",
+ 'plaintext_root_hash': b"anything", }
# optional_inconsistent = { 'crypttext_hash': ('2'*(hashutil.CRYPTO_VAL_SIZE-1), "", 77),
optional_inconsistent = { 'crypttext_hash': (77,),
- 'codec_name': ("digital fountain", ""),
- 'codec_params': ("%d-%d-%d" % (SEGSIZE, K-1, M),
- "%d-%d-%d" % (SEGSIZE-1, K, M),
- "%d-%d-%d" % (SEGSIZE, K, M-1)),
- 'tail_codec_params': ("%d-%d-%d" % (TAIL_SEGSIZE, K-1, M),
- "%d-%d-%d" % (TAIL_SEGSIZE-1, K, M),
- "%d-%d-%d" % (TAIL_SEGSIZE, K, M-1)),
+ 'codec_name': (b"digital fountain", b""),
+ 'codec_params': (b"%d-%d-%d" % (SEGSIZE, K-1, M),
+ b"%d-%d-%d" % (SEGSIZE-1, K, M),
+ b"%d-%d-%d" % (SEGSIZE, K, M-1)),
+ 'tail_codec_params': (b"%d-%d-%d" % (TAIL_SEGSIZE, K-1, M),
+ b"%d-%d-%d" % (TAIL_SEGSIZE-1, K, M),
+ b"%d-%d-%d" % (TAIL_SEGSIZE, K, M-1)),
'num_segments': (NUM_SEGMENTS-1,),
'size': (SIZE-1,),
'needed_shares': (K-1,),
@@ -209,7 +219,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase):
uebhash = hashutil.uri_extension_hash(uebstring)
fb = FakeBucketReaderWriterProxy()
fb.put_uri_extension(uebstring)
- verifycap = uri.CHKFileVerifierURI(storage_index='x'*16, uri_extension_hash=uebhash, needed_shares=self.K, total_shares=self.M, size=self.SIZE)
+ verifycap = uri.CHKFileVerifierURI(storage_index=b'x'*16, uri_extension_hash=uebhash, needed_shares=self.K, total_shares=self.M, size=self.SIZE)
vup = checker.ValidatedExtendedURIProxy(fb, verifycap)
return vup.start()
@@ -232,7 +242,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase):
def test_reject_insufficient(self):
dl = []
- for k in self.mindict.iterkeys():
+ for k in self.mindict.keys():
insuffdict = self.mindict.copy()
del insuffdict[k]
d = self._test_reject(insuffdict)
@@ -241,7 +251,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase):
def test_accept_optional(self):
dl = []
- for k in self.optional_consistent.iterkeys():
+ for k in self.optional_consistent.keys():
mydict = self.mindict.copy()
mydict[k] = self.optional_consistent[k]
d = self._test_accept(mydict)
@@ -250,7 +260,7 @@ class ValidatedExtendedURIProxy(unittest.TestCase):
def test_reject_optional(self):
dl = []
- for k in self.optional_inconsistent.iterkeys():
+ for k in self.optional_inconsistent.keys():
for v in self.optional_inconsistent[k]:
mydict = self.mindict.copy()
mydict[k] = v
@@ -398,7 +408,7 @@ class Roundtrip(GridTestMixin, unittest.TestCase):
self.basedir = self.mktemp()
self.set_up_grid()
self.c0 = self.g.clients[0]
- DATA = "p"*size
+ DATA = b"p"*size
d = self.upload(DATA)
d.addCallback(lambda n: download_to_data(n))
def _downloaded(newdata):
From 7b302871e43764c8af5d79c008559884746d4450 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 18 Sep 2020 11:41:28 -0400
Subject: [PATCH 2/6] Python 2 tests pass again.
---
src/allmydata/immutable/upload.py | 8 ++++----
src/allmydata/test/test_encode.py | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index dbe348a26..b12b90ecc 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -1,4 +1,4 @@
-from past.builtins import long
+from past.builtins import long, unicode
import os, time, weakref, itertools
from zope.interface import implementer
@@ -1500,7 +1500,7 @@ class AssistedUploader(object):
Returns a Deferred that will fire with the UploadResults instance.
"""
- precondition(isinstance(storage_index, str), storage_index)
+ precondition(isinstance(storage_index, bytes), storage_index)
self._started = time.time()
eu = IEncryptedUploadable(encrypted_uploadable)
eu.set_upload_status(self._upload_status)
@@ -1653,7 +1653,7 @@ class BaseUploadable(object):
def set_default_encoding_parameters(self, default_params):
assert isinstance(default_params, dict)
for k,v in default_params.items():
- precondition(isinstance(k, str), k, v)
+ precondition(isinstance(k, (bytes, unicode)), k, v)
precondition(isinstance(v, int), k, v)
if "k" in default_params:
self.default_encoding_param_k = default_params["k"]
@@ -1773,7 +1773,7 @@ class FileName(FileHandle):
then the hash will be hashed together with the string in the
"convergence" argument to form the encryption key.
"""
- assert convergence is None or isinstance(convergence, str), (convergence, type(convergence))
+ assert convergence is None or isinstance(convergence, bytes), (convergence, type(convergence))
FileHandle.__init__(self, open(filename, "rb"), convergence=convergence)
def close(self):
FileHandle.close(self)
diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py
index 7cec5a7d3..8eab6ac45 100644
--- a/src/allmydata/test/test_encode.py
+++ b/src/allmydata/test/test_encode.py
@@ -274,7 +274,7 @@ class Encode(unittest.TestCase):
data = make_data(datalen)
# force use of multiple segments
e = encode.Encoder()
- u = upload.Data(data, convergence="some convergence string")
+ u = upload.Data(data, convergence=b"some convergence string")
u.set_default_encoding_parameters({'max_segment_size': max_segment_size,
'k': 25, 'happy': 75, 'n': 100})
eu = upload.EncryptAnUploadable(u)
@@ -304,7 +304,7 @@ class Encode(unittest.TestCase):
def _check(res):
verifycap = res
- self.failUnless(isinstance(verifycap.uri_extension_hash, str))
+ self.failUnless(isinstance(verifycap.uri_extension_hash, bytes))
self.failUnlessEqual(len(verifycap.uri_extension_hash), 32)
for i,peer in enumerate(all_shareholders):
self.failUnless(peer.closed)
From 050388ee121c7f70901970f924010c1c53a942ed Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 18 Sep 2020 14:30:51 -0400
Subject: [PATCH 3/6] Work better on Python 3, until Nevow is gone.
---
src/allmydata/test/no_network.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index eb2bfd1a1..58289a60a 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -480,10 +480,12 @@ class GridTestMixin(object):
def _record_webports_and_baseurls(self):
self.g._check_clients()
- self.client_webports = [c.getServiceNamed("webish").getPortnum()
- for c in self.g.clients]
- self.client_baseurls = [c.getServiceNamed("webish").getURL()
- for c in self.g.clients]
+ if PY2:
+ # Temporarily disabled on Python 3 until Nevow is gone:
+ self.client_webports = [c.getServiceNamed("webish").getPortnum()
+ for c in self.g.clients]
+ self.client_baseurls = [c.getServiceNamed("webish").getURL()
+ for c in self.g.clients]
def get_client_config(self, i=0):
self.g._check_clients()
From c3bb367a93079744456c76262500b94b4b9e1bb6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 18 Sep 2020 14:31:23 -0400
Subject: [PATCH 4/6] Tests pass on Python 3.
---
src/allmydata/immutable/checker.py | 2 +-
src/allmydata/immutable/literal.py | 5 +++--
src/allmydata/immutable/upload.py | 2 +-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/src/allmydata/immutable/checker.py b/src/allmydata/immutable/checker.py
index 73d6a0263..ce533b969 100644
--- a/src/allmydata/immutable/checker.py
+++ b/src/allmydata/immutable/checker.py
@@ -127,7 +127,7 @@ class ValidatedExtendedURIProxy(object):
# consistent: codec_name, codec_params, tail_codec_params,
# num_segments, size, needed_shares, total_shares
if 'codec_name' in d:
- if d['codec_name'] != "crs":
+ if d['codec_name'] != b"crs":
raise UnsupportedErasureCodec(d['codec_name'])
if 'codec_params' in d:
diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py
index fef4803b1..4832da7c1 100644
--- a/src/allmydata/immutable/literal.py
+++ b/src/allmydata/immutable/literal.py
@@ -1,4 +1,5 @@
-from six.moves import cStringIO as StringIO
+from io import BytesIO
+
from zope.interface import implementer
from twisted.internet import defer
from twisted.internet.interfaces import IPushProducer
@@ -104,7 +105,7 @@ class LiteralFileNode(_ImmutableFileNodeBase):
# vfs.adapters.ftp._FileToConsumerAdapter), neither of which is
# likely to be used as the target for a Tahoe download.
- d = basic.FileSender().beginFileTransfer(StringIO(data), consumer)
+ d = basic.FileSender().beginFileTransfer(BytesIO(data), consumer)
d.addCallback(lambda lastSent: consumer)
return d
diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index b3994310c..884b2cf5e 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -1377,7 +1377,7 @@ class LiteralUploader(object):
self._progress.set_progress_total(size)
return read_this_many_bytes(uploadable, size)
d.addCallback(_got_size)
- d.addCallback(lambda data: uri.LiteralFileURI("".join(data)))
+ d.addCallback(lambda data: uri.LiteralFileURI(b"".join(data)))
d.addCallback(lambda u: u.to_string())
d.addCallback(self._build_results)
return d
From 8ef2252bd4535b3640e0befd4d473186dc91d731 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 18 Sep 2020 14:32:19 -0400
Subject: [PATCH 5/6] Finish porting to Python 3.
---
src/allmydata/test/test_encode.py | 3 +++
src/allmydata/util/_python3.py | 1 +
2 files changed, 4 insertions(+)
diff --git a/src/allmydata/test/test_encode.py b/src/allmydata/test/test_encode.py
index 8eab6ac45..028a988cb 100644
--- a/src/allmydata/test/test_encode.py
+++ b/src/allmydata/test/test_encode.py
@@ -1,3 +1,6 @@
+"""
+Ported to Python 3.
+"""
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index db02f0b17..d16f05a9a 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -85,6 +85,7 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_crypto",
"allmydata.test.test_deferredutil",
"allmydata.test.test_dictutil",
+ "allmydata.test.test_encode",
"allmydata.test.test_encodingutil",
"allmydata.test.test_happiness",
"allmydata.test.test_hashtree",
From a4da6c3dbeb73bdf6c1d5d1154c6a2a6bcb2b7f6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 21 Sep 2020 10:35:56 -0400
Subject: [PATCH 6/6] Clarify comment.
---
src/allmydata/uri.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index bad0dd9df..2c367cafe 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -13,7 +13,8 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
- # Don't import bytes or str, to prevent leaks.
+ # Don't import bytes or str, to prevent future's newbytes leaking and
+ # breaking code that only expects normal bytes.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401
str = unicode