Merge branch 'master' into 3346.abbreviate-and-time_format-py3

misc/python3/ratchet-passing

@@ -28,6 +28,14 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_failure
 allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results
 allmydata.test.test_deferredutil.DeferredUtilTests.test_success
 allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls
+allmydata.test.test_hashtree.Complete.test_create
+allmydata.test.test_hashtree.Complete.test_dump
+allmydata.test.test_hashtree.Complete.test_needed_hashes
+allmydata.test.test_hashtree.Incomplete.test_check
+allmydata.test.test_hashtree.Incomplete.test_create
+allmydata.test.test_hashtree.Incomplete.test_depth_of
+allmydata.test.test_hashtree.Incomplete.test_large
+allmydata.test.test_hashtree.Incomplete.test_needed_hashes
 allmydata.test.test_hashutil.HashUtilTests.test_chk
 allmydata.test.test_hashutil.HashUtilTests.test_hashers
 allmydata.test.test_hashutil.HashUtilTests.test_known_answers

newsfragments/3354.minor

@@ -0,0 +1 @@
+

src/allmydata/hashtree.py

@@ -1,7 +1,4 @@
 # -*- test-case-name: allmydata.test.test_hashtree -*-
-
-from allmydata.util import mathutil # from the pyutil library
-
 """
 Read and write chunks from files.
 
@@ -50,6 +47,17 @@ or implied.  It probably won't make your computer catch on fire,
 or eat  your children, but it might.  Use at your own risk.
 """
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
+from allmydata.util import mathutil # from the pyutil library
+
 from allmydata.util import base32
 from allmydata.util.hashutil import tagged_hash, tagged_pair_hash
 
@@ -170,9 +178,10 @@ def depth_of(i):
     return mathutil.log_floor(i+1, 2)
 
 def empty_leaf_hash(i):
-    return tagged_hash('Merkle tree empty leaf', "%d" % i)
+    return tagged_hash(b'Merkle tree empty leaf', b"%d" % i)
+
 def pair_hash(a, b):
-    return tagged_pair_hash('Merkle tree internal node', a, b)
+    return tagged_pair_hash(b'Merkle tree internal node', a, b)
 
 class HashTree(CompleteBinaryTreeMixin, list):
     """
@@ -215,7 +224,7 @@ class HashTree(CompleteBinaryTreeMixin, list):
         while len(rows[-1]) != 1:
             last = rows[-1]
             rows += [[pair_hash(last[2*i], last[2*i+1])
-                                for i in xrange(len(last)//2)]]
+                                for i in range(len(last)//2)]]
         # Flatten the list of rows into a single list.
         rows.reverse()
         self[:] = sum(rows, [])
@@ -289,7 +298,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
         rows = [L]
         while len(rows[-1]) != 1:
             last = rows[-1]
-            rows += [[None for i in xrange(len(last)//2)]]
+            rows += [[None for i in range(len(last)//2)]]
         # Flatten the list of rows into a single list.
         rows.reverse()
         self[:] = sum(rows, [])
@@ -372,12 +381,12 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
 
         assert isinstance(hashes, dict)
         for h in hashes.values():
-            assert isinstance(h, str)
+            assert isinstance(h, bytes)
         assert isinstance(leaves, dict)
         for h in leaves.values():
-            assert isinstance(h, str)
+            assert isinstance(h, bytes)
         new_hashes = hashes.copy()
-        for leafnum,leafhash in leaves.iteritems():
+        for leafnum,leafhash in leaves.items():
             hashnum = self.first_leaf_num + leafnum
             if hashnum in new_hashes:
                 if new_hashes[hashnum] != leafhash:
@@ -416,7 +425,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
 
             # first we provisionally add all hashes to the tree, comparing
             # any duplicates
-            for i,h in new_hashes.iteritems():
+            for i,h in new_hashes.items():
                 if self[i]:
                     if self[i] != h:
                         raise BadHashError("new hash %s does not match "

src/allmydata/test/test_hashtree.py

@@ -1,4 +1,18 @@
-# -*- test-case-name: allmydata.test.test_hashtree -*-
+"""
+Tests for allmydata.hashtree.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min  # noqa: F401
+
 
 from twisted.trial import unittest
 
@@ -7,8 +21,8 @@ from allmydata import hashtree
 
 
 def make_tree(numleaves):
-    leaves = ["%d" % i for i in range(numleaves)]
+    leaves = [b"%d" % i for i in range(numleaves)]
-    leaf_hashes = [tagged_hash("tag", leaf) for leaf in leaves]
+    leaf_hashes = [tagged_hash(b"tag", leaf) for leaf in leaves]
     ht = hashtree.HashTree(leaf_hashes)
     return ht
 
@@ -20,7 +34,7 @@ class Complete(unittest.TestCase):
         ht = make_tree(8)
         root = ht[0]
         self.failUnlessEqual(len(root), 32)
-        self.failUnlessEqual(ht.get_leaf(0), tagged_hash("tag", "0"))
+        self.failUnlessEqual(ht.get_leaf(0), tagged_hash(b"tag", b"0"))
         self.failUnlessRaises(IndexError, ht.get_leaf, 8)
         self.failUnlessEqual(ht.get_leaf_index(0), 7)
         self.failUnlessRaises(IndexError, ht.parent, 0)
@@ -143,7 +157,7 @@ class Incomplete(unittest.TestCase):
         current_hashes = list(iht)
         # this should fail because there aren't enough hashes known
         try:
-            iht.set_hashes(leaves={0: tagged_hash("tag", "0")})
+            iht.set_hashes(leaves={0: tagged_hash(b"tag", b"0")})
         except hashtree.NotEnoughHashesError:
             pass
         else:
@@ -157,7 +171,7 @@ class Incomplete(unittest.TestCase):
         chain = {0: ht[0], 2: ht[2], 4: ht[4], 8: ht[8]}
         # this should fail because the leaf hash is just plain wrong
         try:
-            iht.set_hashes(chain, leaves={0: tagged_hash("bad tag", "0")})
+            iht.set_hashes(chain, leaves={0: tagged_hash(b"bad tag", b"0")})
         except hashtree.BadHashError:
             pass
         else:
@@ -166,18 +180,18 @@ class Incomplete(unittest.TestCase):
         # this should fail because we give it conflicting hashes: one as an
         # internal node, another as a leaf
         try:
-            iht.set_hashes(chain, leaves={1: tagged_hash("bad tag", "1")})
+            iht.set_hashes(chain, leaves={1: tagged_hash(b"bad tag", b"1")})
         except hashtree.BadHashError:
             pass
         else:
             self.fail("didn't catch bad hash")
 
         bad_chain = chain.copy()
-        bad_chain[2] = ht[2] + "BOGUS"
+        bad_chain[2] = ht[2] + b"BOGUS"
 
         # this should fail because the internal hash is wrong
         try:
-            iht.set_hashes(bad_chain, leaves={0: tagged_hash("tag", "0")})
+            iht.set_hashes(bad_chain, leaves={0: tagged_hash(b"tag", b"0")})
         except hashtree.BadHashError:
             pass
         else:
@@ -185,23 +199,23 @@ class Incomplete(unittest.TestCase):
 
         # this should succeed
         try:
-            iht.set_hashes(chain, leaves={0: tagged_hash("tag", "0")})
+            iht.set_hashes(chain, leaves={0: tagged_hash(b"tag", b"0")})
         except hashtree.BadHashError as e:
             self.fail("bad hash: %s" % e)
 
-        self.failUnlessEqual(ht.get_leaf(0), tagged_hash("tag", "0"))
+        self.failUnlessEqual(ht.get_leaf(0), tagged_hash(b"tag", b"0"))
         self.failUnlessRaises(IndexError, ht.get_leaf, 8)
 
         # this should succeed too
         try:
-            iht.set_hashes(leaves={1: tagged_hash("tag", "1")})
+            iht.set_hashes(leaves={1: tagged_hash(b"tag", b"1")})
         except hashtree.BadHashError:
             self.fail("bad hash")
 
         # this should fail because we give it hashes that conflict with some
         # that we added successfully before
         try:
-            iht.set_hashes(leaves={1: tagged_hash("bad tag", "1")})
+            iht.set_hashes(leaves={1: tagged_hash(b"bad tag", b"1")})
         except hashtree.BadHashError:
             pass
         else:
@@ -214,6 +228,6 @@ class Incomplete(unittest.TestCase):
 
         # this should succeed
         try:
-            iht.set_hashes(chain, leaves={4: tagged_hash("tag", "4")})
+            iht.set_hashes(chain, leaves={4: tagged_hash(b"tag", b"4")})
         except hashtree.BadHashError as e:
             self.fail("bad hash: %s" % e)

src/allmydata/util/_python3.py

@@ -15,6 +15,7 @@ if PY2:
 
 # Keep these sorted alphabetically, to reduce merge conflicts:
 PORTED_MODULES = [
+    "allmydata.hashtree",
     "allmydata.util.abbreviate",
     "allmydata.util.assertutil",
     "allmydata.util.base32",
@@ -36,6 +37,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
     "allmydata.test.test_deferredutil",
+    "allmydata.test.test_hashtree",
     "allmydata.test.test_hashutil",
     "allmydata.test.test_humanreadable",
     "allmydata.test.test_netstring",