From 1931022ff0509d0711b872b465ba31593737eff2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:40:10 -0400 Subject: [PATCH 1/6] News fragment. --- newsfragments/4062.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4062.minor diff --git a/newsfragments/4062.minor b/newsfragments/4062.minor new file mode 100644 index 000000000..e69de29bb From a8b68c217f8880f5493e4cb349293c3600d57d37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:40:22 -0400 Subject: [PATCH 2/6] Update to new mypy and Twisted for type checking. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 67a089b0c..294f26284 100644 --- a/tox.ini +++ b/tox.ini @@ -125,7 +125,7 @@ commands = [testenv:typechecks] basepython = python3 deps = - mypy==1.4.1 + mypy==1.5.1 mypy-zope types-mock types-six @@ -134,7 +134,7 @@ deps = types-pyOpenSSL foolscap # Upgrade when new releases come out: - Twisted==22.10.0 + Twisted==23.8.0 commands = mypy src From e96d67f541c9a5ad8bbbb7fb392f8ae5bf9063cd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 09:44:48 -0400 Subject: [PATCH 3/6] Upgrade ruff. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 294f26284..027736c1a 100644 --- a/tox.ini +++ b/tox.ini @@ -99,9 +99,10 @@ skip_install = true deps = # Pin a specific version so we get consistent outcomes; update this # occasionally: - ruff == 0.0.278 + ruff == 0.0.287 # towncrier doesn't work with importlib_resources 6.0.0 # https://github.com/twisted/towncrier/issues/528 + # Will be fixed in first version of Towncrier that is larger than 2023.6. importlib_resources < 6.0.0 towncrier # On macOS, git inside of towncrier needs $HOME. From 7d7ca29e3d502c3e248bf49572501c8431c39aee Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 5 Sep 2023 10:14:10 -0400 Subject: [PATCH 4/6] Work around Hypothesis complaint about differing executors due to subclassing. See https://hypothesis.readthedocs.io/en/latest/settings.html#health-checks --- newsfragments/4063.minor | 0 src/allmydata/test/test_storage_http.py | 20 ++++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 newsfragments/4063.minor diff --git a/newsfragments/4063.minor b/newsfragments/4063.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index bc266e824..eaed5f07c 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -1673,10 +1673,12 @@ class SharedImmutableMutableTestsMixin: # semantically valid under HTTP. check_bad_range("bytes=0-") - @given(data_length=st.integers(min_value=1, max_value=300000)) - def test_read_with_no_range(self, data_length): + def _read_with_no_range_test(self, data_length): """ A read with no range returns the whole mutable/immutable. + + Actual test is defined in subclasses, to fix complaints from Hypothesis + about the method having different executors. """ storage_index, uploaded_data, _ = self.upload(1, data_length) response = self.http.result_of_with_flush( @@ -1770,6 +1772,13 @@ class ImmutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def get_leases(self, storage_index): return self.http.storage_server.get_leases(storage_index) + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole immutable. + """ + return self._read_with_no_range_test(data_length) + class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): """Shared tests, running on mutables.""" @@ -1809,3 +1818,10 @@ class MutableSharedTests(SharedImmutableMutableTestsMixin, SyncTestCase): def get_leases(self, storage_index): return self.http.storage_server.get_slot_leases(storage_index) + + @given(data_length=st.integers(min_value=1, max_value=300000)) + def test_read_with_no_range(self, data_length): + """ + A read with no range returns the whole mutable. + """ + return self._read_with_no_range_test(data_length) From 6d626851bfbae700dd66fec2e66d320698325ff7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Sep 2023 09:26:30 -0400 Subject: [PATCH 5/6] news fragment --- newsfragments/4066.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4066.minor diff --git a/newsfragments/4066.minor b/newsfragments/4066.minor new file mode 100644 index 000000000..e69de29bb From d6b38bc7a22cd5095fa086648d6a5bf9b833857f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Sep 2023 09:26:35 -0400 Subject: [PATCH 6/6] test vectors --- src/allmydata/test/test_hashtree.py | 32 +++++++++++++++-------------- src/allmydata/test/test_hashutil.py | 28 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index 5abe2095e..d9be96bd5 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -1,32 +1,22 @@ """ Tests for allmydata.hashtree. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import annotations -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - - -from twisted.trial import unittest +from .common import SyncTestCase +from base64 import b32encode from allmydata.util.hashutil import tagged_hash from allmydata import hashtree - def make_tree(numleaves): leaves = [b"%d" % i for i in range(numleaves)] leaf_hashes = [tagged_hash(b"tag", leaf) for leaf in leaves] ht = hashtree.HashTree(leaf_hashes) return ht -class Complete(unittest.TestCase): +class Complete(SyncTestCase): def test_create(self): # try out various sizes, since we pad to a power of two ht = make_tree(6) @@ -40,6 +30,18 @@ class Complete(unittest.TestCase): self.failUnlessRaises(IndexError, ht.parent, 0) self.failUnlessRaises(IndexError, ht.needed_for, -1) + def test_well_known_tree(self): + self.assertEqual( + [b32encode(s).strip(b"=").lower() for s in make_tree(3)], + [b'vxuqudnucceja4pqkdqy5txapagxubm5moupzqywkbg2jrjkaola', + b'weycjri4jlcaunca2jyx2kr7sbtb7qdriog3f26g5jpc5awfeazq', + b'5ovy3g2wwjnxoqtja4licckxkbqjef4xsjtclk6gxnsl66kvow6a', + b'esd34nbzri75l3j2vwetpk3dvlvsxstkbaktomonrulpks3df3sq', + b'jkxbwa2tppyfax35o72tbjecxvaa4xphma6zbyfbkkku3ed2657a', + b'wfisavaqgab2raihe7dld2qjps4rtxyiubgfs5enziokey2msjwa', + b't3kza5vwx3tlowdemmgdyigp62ju57qduyfh7uulnfkc7mj2ncrq'], + ) + def test_needed_hashes(self): ht = make_tree(8) self.failUnlessEqual(ht.needed_hashes(0), set([8, 4, 2])) @@ -65,7 +67,7 @@ class Complete(unittest.TestCase): self.failUnless("\n 8:" in d) self.failUnless("\n 4:" in d) -class Incomplete(unittest.TestCase): +class Incomplete(SyncTestCase): def test_create(self): ht = hashtree.IncompleteHashTree(6) diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index 482e79c0b..1efa2d9db 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -44,6 +44,34 @@ class HashUtilTests(unittest.TestCase): self.failUnlessEqual(len(h2), 16) self.failUnlessEqual(h1, h2) + def test_well_known_tagged_hash(self): + self.assertEqual( + b"yra322btzoqjp4ts2jon5dztgnilcdg6jgztgk7joi6qpjkitg2q", + base32.b2a(hashutil.tagged_hash(b"tag", b"hello world")), + ) + self.assertEqual( + b"kfbsfssrv2bvtp3regne6j7gpdjcdjwncewriyfdtt764o5oa7ta", + base32.b2a(hashutil.tagged_hash(b"different", b"hello world")), + ) + self.assertEqual( + b"z34pzkgo36chbjz2qykonlxthc4zdqqquapw4bcaoogzvmmcr3zq", + base32.b2a(hashutil.tagged_hash(b"different", b"goodbye world")), + ) + + def test_well_known_tagged_pair_hash(self): + self.assertEqual( + b"wmto44q3shtezwggku2fxztfkwibvznkfu6clatnvfog527sb6dq", + base32.b2a(hashutil.tagged_pair_hash(b"tag", b"hello", b"world")), + ) + self.assertEqual( + b"lzn27njx246jhijpendqrxlk4yb23nznbcrihommbymg5e7quh4a", + base32.b2a(hashutil.tagged_pair_hash(b"different", b"hello", b"world")), + ) + self.assertEqual( + b"qnehpoypxxdhjheqq7dayloghtu42yr55uylc776zt23ii73o3oq", + base32.b2a(hashutil.tagged_pair_hash(b"different", b"goodbye", b"world")), + ) + def test_chk(self): h1 = hashutil.convergence_hash(3, 10, 1000, b"data", b"secret") h2 = hashutil.convergence_hasher(3, 10, 1000, b"secret")