From 9bdc085c26a21f25810bd8bca0be2d3b55bf230f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Aug 2020 13:20:56 -0400 Subject: [PATCH 1/7] news fragment --- newsfragments/3388.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3388.minor diff --git a/newsfragments/3388.minor b/newsfragments/3388.minor new file mode 100644 index 000000000..e69de29bb From 60759597f37e9748eb3ccf3ca753f96034086afd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Aug 2020 13:21:48 -0400 Subject: [PATCH 2/7] Ask for a particular version of Python In particular, a version Tahoe-LAFS supports. --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f7381f33..c15eb1746 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -508,6 +508,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "20.04" + PYTHON_VERSION: "2.7" build-image-centos-8: From a9fa70c3d5839ce5b22bf28dd5972bd0cca37f02 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 21 Aug 2020 15:53:58 -0400 Subject: [PATCH 3/7] Add upper bound for `attrs` library --- newsfragments/3389.minor | 0 setup.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3389.minor diff --git a/newsfragments/3389.minor b/newsfragments/3389.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.py b/setup.py index a35023b8b..db5a5490d 100644 --- a/setup.py +++ b/setup.py @@ -117,7 +117,8 @@ install_requires = [ "eliot ~= 1.7", # A great way to define types of values. - "attrs >= 18.2.0", + # XXX: drop the upper bound: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3390 + "attrs >= 18.2.0, < 20", # WebSocket library for twisted and asyncio "autobahn >= 19.5.2", From e22bed447bc4b3ccaf29b998fb8a8a72a418bef0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Aug 2020 13:10:17 -0400 Subject: [PATCH 4/7] Some updates for URI tests. --- src/allmydata/test/test_uri.py | 87 +++++++++++++++++++++------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index c04b1259d..a14e0c304 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -1,4 +1,17 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + import os from twisted.trial import unittest from allmydata import uri @@ -40,24 +53,24 @@ class Literal(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(u.get_verify_cap(), None) def test_empty(self): - data = "" # This data is some *very* small data! + data = b"" # This data is some *very* small data! return self._help_test(data) def test_pack(self): - data = "This is some small data" + data = b"This is some small data" return self._help_test(data) def test_nonascii(self): - data = "This contains \x00 and URI:LIT: and \n, oh my." + data = b"This contains \x00 and URI:LIT: and \n, oh my." return self._help_test(data) class Compare(testutil.ReallyEqualMixin, unittest.TestCase): def test_compare(self): - lit1 = uri.LiteralFileURI("some data") + lit1 = uri.LiteralFileURI(b"some data") fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834' chk1 = uri.CHKFileURI.init_from_string(fileURI) chk2 = uri.CHKFileURI.init_from_string(fileURI) - unk = uri.UnknownURI("lafs://from_the_future") + unk = uri.UnknownURI(b"lafs://from_the_future") self.failIfEqual(lit1, chk1) self.failUnlessReallyEqual(chk1, chk2) self.failIfEqual(chk1, "not actually a URI") @@ -66,12 +79,12 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(len(s), 3) # since chk1==chk2 def test_is_uri(self): - lit1 = uri.LiteralFileURI("some data").to_string() + lit1 = uri.LiteralFileURI(b"some data").to_string() self.failUnless(uri.is_uri(lit1)) self.failIf(uri.is_uri(None)) def test_is_literal_file_uri(self): - lit1 = uri.LiteralFileURI("some data").to_string() + lit1 = uri.LiteralFileURI(b"some data").to_string() self.failUnless(uri.is_literal_file_uri(lit1)) self.failIf(uri.is_literal_file_uri(None)) self.failIf(uri.is_literal_file_uri("foo")) @@ -89,9 +102,9 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase): class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) - uri_extension_hash = hashutil.uri_extension_hash("stuff") + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -145,8 +158,8 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): v2 = uri.from_string(v.to_string()) self.failUnlessReallyEqual(v, v2) - v3 = uri.CHKFileVerifierURI(storage_index="\x00"*16, - uri_extension_hash="\x00"*32, + v3 = uri.CHKFileVerifierURI(storage_index=b"\x00"*16, + uri_extension_hash=b"\x00"*32, needed_shares=3, total_shares=10, size=1234) @@ -155,9 +168,9 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): self.failIf(v3.is_mutable()) def test_pack_badly(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) - uri_extension_hash = hashutil.uri_extension_hash("stuff") + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 25 total_shares = 100 size = 1234 @@ -186,23 +199,23 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): class Extension(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - data = {"stuff": "value", + data = {"stuff": b"value", "size": 12, "needed_shares": 3, - "big_hash": hashutil.tagged_hash("foo", "bar"), + "big_hash": hashutil.tagged_hash(b"foo", b"bar"), } ext = uri.pack_extension(data) d = uri.unpack_extension(ext) - self.failUnlessReallyEqual(d["stuff"], "value") + self.failUnlessReallyEqual(d["stuff"], b"value") self.failUnlessReallyEqual(d["size"], 12) - self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash("foo", "bar")) + self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash(b"foo", b"bar")) readable = uri.unpack_extension_readable(ext) self.failUnlessReallyEqual(readable["needed_shares"], 3) - self.failUnlessReallyEqual(readable["stuff"], "value") + self.failUnlessReallyEqual(readable["stuff"], b"value") self.failUnlessReallyEqual(readable["size"], 12) self.failUnlessReallyEqual(readable["big_hash"], - base32.b2a(hashutil.tagged_hash("foo", "bar"))) + base32.b2a(hashutil.tagged_hash(b"foo", b"bar"))) self.failUnlessReallyEqual(readable["UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext))) @@ -222,7 +235,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(isinstance(u2.get_error(), CapConstraintError)) # Future caps might have non-ASCII chars in them. (Or maybe not, who can tell about the future?) - future_uri = u"I am a cap from the \u263A future. Whatever you ".encode('utf-8') + future_uri = u"I am a cap from the \u263A future. Whatever you " u = uri.from_string(future_uri) self.failUnless(isinstance(u, uri.UnknownURI)) self.failUnlessReallyEqual(u.to_string(), future_uri) @@ -243,8 +256,8 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase): class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): def setUp(self): - self.writekey = "\x01" * 16 - self.fingerprint = "\x02" * 32 + self.writekey = b"\x01" * 16 + self.fingerprint = b"\x02" * 32 self.readkey = hashutil.ssk_readkey_hash(self.writekey) self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) @@ -417,7 +430,13 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): self.failIf(u2.is_readonly()) self.failUnless(u2.is_mutable()) - cap3 = cap+":"+os.urandom(40) # parse *that*! + + cap3 = cap+":" + for item in os.urandom(40): + if isinstance(item, int): + cap3 += chr(item) + else: + cap3 += chr(ord(item)) u3 = uri.WriteableMDMFFileURI.init_from_string(cap3) self.failUnlessReallyEqual(self.writekey, u3.writekey) self.failUnlessReallyEqual(self.fingerprint, u3.fingerprint) @@ -468,8 +487,8 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 n = uri.WriteableSSKFileURI(writekey, fingerprint) u1 = uri.DirectoryURI(n) @@ -536,8 +555,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): u1.get_verify_cap()._filenode_uri) def test_immutable(self): - readkey = "\x01" * 16 - uri_extension_hash = hashutil.uri_extension_hash("stuff") + readkey = b"\x01" * 16 + uri_extension_hash = hashutil.uri_extension_hash(b"stuff") needed_shares = 3 total_shares = 10 size = 1234 @@ -597,7 +616,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(str(u2_verifier)) def test_literal(self): - u0 = uri.LiteralFileURI("data") + u0 = uri.LiteralFileURI(b"data") u1 = uri.LiteralDirectoryURI(u0) self.failUnless(str(u1)) self.failUnlessReallyEqual(u1.to_string(), "URI:DIR2-LIT:mrqxiyi") @@ -611,8 +630,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(u1.abbrev_si(), "") def test_mdmf(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) self.failIf(d1.is_readonly()) @@ -635,8 +654,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessIsInstance(d3, uri.UnknownURI) def test_mdmf_attenuation(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) @@ -676,8 +695,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): def test_mdmf_verifier(self): # I'm not sure what I want to write here yet. - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) v1 = d1.get_verify_cap() diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 794edef40..84c08a36c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -82,6 +82,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_spans", "allmydata.test.test_statistics", "allmydata.test.test_time_format", + "allmydata.test.test_uri", "allmydata.test.test_util", "allmydata.test.test_version", ] From 388f27d2066a2611e353fe5a4786a89daf778495 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Aug 2020 13:41:58 -0400 Subject: [PATCH 5/7] Some more porting, tests pass on Python 2. --- src/allmydata/test/test_uri.py | 42 +++---- src/allmydata/uri.py | 221 +++++++++++++++++---------------- 2 files changed, 137 insertions(+), 126 deletions(-) diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index a14e0c304..4b9c229f0 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -67,7 +67,7 @@ class Literal(testutil.ReallyEqualMixin, unittest.TestCase): class Compare(testutil.ReallyEqualMixin, unittest.TestCase): def test_compare(self): lit1 = uri.LiteralFileURI(b"some data") - fileURI = 'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834' + fileURI = b'URI:CHK:f5ahxa25t4qkktywz6teyfvcx4:opuioq7tj2y6idzfp6cazehtmgs5fdcebcz3cygrxyydvcozrmeq:3:10:345834' chk1 = uri.CHKFileURI.init_from_string(fileURI) chk2 = uri.CHKFileURI.init_from_string(fileURI) unk = uri.UnknownURI(b"lafs://from_the_future") @@ -89,11 +89,14 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase): self.failIf(uri.is_literal_file_uri(None)) self.failIf(uri.is_literal_file_uri("foo")) self.failIf(uri.is_literal_file_uri("ro.foo")) - self.failIf(uri.is_literal_file_uri("URI:LITfoo")) + self.failIf(uri.is_literal_file_uri(b"URI:LITfoo")) self.failUnless(uri.is_literal_file_uri("ro.URI:LIT:foo")) self.failUnless(uri.is_literal_file_uri("imm.URI:LIT:foo")) def test_has_uri_prefix(self): + self.failUnless(uri.has_uri_prefix(b"URI:foo")) + self.failUnless(uri.has_uri_prefix(b"ro.URI:foo")) + self.failUnless(uri.has_uri_prefix(b"imm.URI:foo")) self.failUnless(uri.has_uri_prefix("URI:foo")) self.failUnless(uri.has_uri_prefix("ro.URI:foo")) self.failUnless(uri.has_uri_prefix("imm.URI:foo")) @@ -151,7 +154,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(u.to_string(), u2imm.to_string()) v = u.get_verify_cap() - self.failUnless(isinstance(v.to_string(), str)) + self.failUnless(isinstance(v.to_string(), bytes)) self.failUnless(v.is_readonly()) self.failIf(v.is_mutable()) @@ -163,7 +166,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): needed_shares=3, total_shares=10, size=1234) - self.failUnless(isinstance(v3.to_string(), str)) + self.failUnless(isinstance(v3.to_string(), bytes)) self.failUnless(v3.is_readonly()) self.failIf(v3.is_mutable()) @@ -222,12 +225,14 @@ class Extension(testutil.ReallyEqualMixin, unittest.TestCase): class Unknown(testutil.ReallyEqualMixin, unittest.TestCase): def test_from_future(self): # any URI type that we don't recognize should be treated as unknown - future_uri = "I am a URI from the future. Whatever you do, don't " + future_uri = b"I am a URI from the future. Whatever you do, don't " u = uri.from_string(future_uri) self.failUnless(isinstance(u, uri.UnknownURI)) self.failUnlessReallyEqual(u.to_string(), future_uri) self.failUnless(u.get_readonly() is None) self.failUnless(u.get_error() is None) + future_uri_unicode = future_uri.decode("utf-8") + self.assertEqual(future_uri, uri.from_string(future_uri_unicode).to_string()) u2 = uri.UnknownURI(future_uri, error=CapConstraintError("...")) self.failUnlessReallyEqual(u.to_string(), future_uri) @@ -235,7 +240,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(isinstance(u2.get_error(), CapConstraintError)) # Future caps might have non-ASCII chars in them. (Or maybe not, who can tell about the future?) - future_uri = u"I am a cap from the \u263A future. Whatever you " + future_uri = u"I am a cap from the \u263A future. Whatever you ".encode("utf-8") u = uri.from_string(future_uri) self.failUnless(isinstance(u, uri.UnknownURI)) self.failUnlessReallyEqual(u.to_string(), future_uri) @@ -251,7 +256,7 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase): def test_constraint(self): bad = "http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/" self.failUnlessRaises(uri.BadURIError, uri.DirectoryURI.init_from_string, bad) - fileURI = 'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834' + fileURI = b'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834' uri.CHKFileURI.init_from_string(fileURI) class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): @@ -423,7 +428,7 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): u1 = uri.WriteableMDMFFileURI(self.writekey, self.fingerprint) cap = u1.to_string() - cap2 = cap+":I COME FROM THE FUTURE" + cap2 = cap+b":I COME FROM THE FUTURE" u2 = uri.WriteableMDMFFileURI.init_from_string(cap2) self.failUnlessReallyEqual(self.writekey, u2.writekey) self.failUnlessReallyEqual(self.fingerprint, u2.fingerprint) @@ -431,26 +436,21 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(u2.is_mutable()) - cap3 = cap+":" - for item in os.urandom(40): - if isinstance(item, int): - cap3 += chr(item) - else: - cap3 += chr(ord(item)) + cap3 = cap+b":" + os.urandom(40) u3 = uri.WriteableMDMFFileURI.init_from_string(cap3) self.failUnlessReallyEqual(self.writekey, u3.writekey) self.failUnlessReallyEqual(self.fingerprint, u3.fingerprint) self.failIf(u3.is_readonly()) self.failUnless(u3.is_mutable()) - cap4 = u1.get_readonly().to_string()+":ooh scary future stuff" + cap4 = u1.get_readonly().to_string()+b":ooh scary future stuff" u4 = uri.from_string_mutable_filenode(cap4) self.failUnlessReallyEqual(self.readkey, u4.readkey) self.failUnlessReallyEqual(self.fingerprint, u4.fingerprint) self.failUnless(u4.is_readonly()) self.failUnless(u4.is_mutable()) - cap5 = u1.get_verify_cap().to_string()+":spoilers!" + cap5 = u1.get_verify_cap().to_string()+b":spoilers!" u5 = uri.from_string(cap5) self.failUnlessReallyEqual(self.storage_index, u5.storage_index) self.failUnlessReallyEqual(self.fingerprint, u5.fingerprint) @@ -567,7 +567,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): total_shares=total_shares, size=size) fncap = fnuri.to_string() - self.failUnlessReallyEqual(fncap, "URI:CHK:aeaqcaibaeaqcaibaeaqcaibae:nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa:3:10:1234") + self.failUnlessReallyEqual(fncap, b"URI:CHK:aeaqcaibaeaqcaibaeaqcaibae:nf3nimquen7aeqm36ekgxomalstenpkvsdmf6fplj7swdatbv5oa:3:10:1234") u1 = uri.ImmutableDirectoryURI(fnuri) self.failUnless(u1.is_readonly()) self.failIf(u1.is_mutable()) @@ -606,20 +606,20 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(IVerifierURI.providedBy(u2_verifier)) u2vs = u2_verifier.to_string() # URI:DIR2-CHK-Verifier:$key:$ueb:$k:$n:$size - self.failUnless(u2vs.startswith("URI:DIR2-CHK-Verifier:"), u2vs) + self.failUnless(u2vs.startswith(b"URI:DIR2-CHK-Verifier:"), u2vs) u2_verifier_fileuri = u2_verifier.get_filenode_cap() self.failUnless(IVerifierURI.providedBy(u2_verifier_fileuri)) u2vfs = u2_verifier_fileuri.to_string() # URI:CHK-Verifier:$key:$ueb:$k:$n:$size self.failUnlessReallyEqual(u2vfs, fnuri.get_verify_cap().to_string()) - self.failUnlessReallyEqual(u2vs[len("URI:DIR2-"):], u2vfs[len("URI:"):]) + self.failUnlessReallyEqual(u2vs[len(b"URI:DIR2-"):], u2vfs[len(b"URI:"):]) self.failUnless(str(u2_verifier)) def test_literal(self): u0 = uri.LiteralFileURI(b"data") u1 = uri.LiteralDirectoryURI(u0) self.failUnless(str(u1)) - self.failUnlessReallyEqual(u1.to_string(), "URI:DIR2-LIT:mrqxiyi") + self.failUnlessReallyEqual(u1.to_string(), b"URI:DIR2-LIT:mrqxiyi") self.failUnless(u1.is_readonly()) self.failIf(u1.is_mutable()) self.failUnless(IURI.providedBy(u1)) @@ -627,7 +627,7 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnless(IDirnodeURI.providedBy(u1)) self.failUnlessReallyEqual(u1.get_verify_cap(), None) self.failUnlessReallyEqual(u1.get_storage_index(), None) - self.failUnlessReallyEqual(u1.abbrev_si(), "") + self.failUnlessReallyEqual(u1.abbrev_si(), b"") def test_mdmf(self): writekey = b"\x01" * 16 diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 051b45f79..70fd80c92 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -1,3 +1,4 @@ +from past.builtins import unicode, long import re @@ -24,10 +25,10 @@ class BadURIError(CapConstraintError): # - make variable and method names consistently use _uri for an URI string, # and _cap for a Cap object (decoded URI) -BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) -BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) +BASE32STR_128bits = b'(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) +BASE32STR_256bits = b'(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) -NUMBER='([0-9]+)' +NUMBER=b'([0-9]+)' class _BaseURI(object): @@ -53,10 +54,10 @@ class _BaseURI(object): @implementer(IURI, IImmutableFileURI) class CHKFileURI(_BaseURI): - BASE_STRING='URI:CHK:' - STRING_RE=re.compile('^URI:CHK:'+BASE32STR_128bits+':'+ - BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER+ - '$') + BASE_STRING=b'URI:CHK:' + STRING_RE=re.compile(b'^URI:CHK:'+BASE32STR_128bits+b':'+ + BASE32STR_256bits+b':'+NUMBER+b':'+NUMBER+b':'+NUMBER+ + b'$') def __init__(self, key, uri_extension_hash, needed_shares, total_shares, size): @@ -82,7 +83,7 @@ class CHKFileURI(_BaseURI): assert isinstance(self.total_shares, int) assert isinstance(self.size, (int,long)) - return ('URI:CHK:%s:%s:%d:%d:%d' % + return (b'URI:CHK:%s:%s:%d:%d:%d' % (base32.b2a(self.key), base32.b2a(self.uri_extension_hash), self.needed_shares, @@ -112,9 +113,9 @@ class CHKFileURI(_BaseURI): @implementer(IVerifierURI) class CHKFileVerifierURI(_BaseURI): - BASE_STRING='URI:CHK-Verifier:' - STRING_RE=re.compile('^URI:CHK-Verifier:'+BASE32STR_128bits+':'+ - BASE32STR_256bits+':'+NUMBER+':'+NUMBER+':'+NUMBER) + BASE_STRING=b'URI:CHK-Verifier:' + STRING_RE=re.compile(b'^URI:CHK-Verifier:'+BASE32STR_128bits+b':'+ + BASE32STR_256bits+b':'+NUMBER+b':'+NUMBER+b':'+NUMBER) def __init__(self, storage_index, uri_extension_hash, needed_shares, total_shares, size): @@ -138,7 +139,7 @@ class CHKFileVerifierURI(_BaseURI): assert isinstance(self.total_shares, int) assert isinstance(self.size, (int,long)) - return ('URI:CHK-Verifier:%s:%s:%d:%d:%d' % + return (b'URI:CHK-Verifier:%s:%s:%d:%d:%d' % (si_b2a(self.storage_index), base32.b2a(self.uri_extension_hash), self.needed_shares, @@ -161,8 +162,8 @@ class CHKFileVerifierURI(_BaseURI): @implementer(IURI, IImmutableFileURI) class LiteralFileURI(_BaseURI): - BASE_STRING='URI:LIT:' - STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$') + BASE_STRING=b'URI:LIT:' + STRING_RE=re.compile(b'^URI:LIT:'+base32.BASE32STR_anybytes+b'$') def __init__(self, data=None): if data is not None: @@ -177,7 +178,7 @@ class LiteralFileURI(_BaseURI): return cls(base32.a2b(mo.group(1))) def to_string(self): - return 'URI:LIT:%s' % base32.b2a(self.data) + return b'URI:LIT:%s' % base32.b2a(self.data) def is_readonly(self): return True @@ -202,9 +203,9 @@ class LiteralFileURI(_BaseURI): @implementer(IURI, IMutableFileURI) class WriteableSSKFileURI(_BaseURI): - BASE_STRING='URI:SSK:' - STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+ - BASE32STR_256bits+'$') + BASE_STRING=b'URI:SSK:' + STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+ + BASE32STR_256bits+b'$') def __init__(self, writekey, fingerprint): self.writekey = writekey @@ -223,8 +224,8 @@ class WriteableSSKFileURI(_BaseURI): def to_string(self): assert isinstance(self.writekey, str) assert isinstance(self.fingerprint, str) - return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey), - base32.b2a(self.fingerprint)) + return b'URI:SSK:%s:%s' % (base32.b2a(self.writekey), + base32.b2a(self.fingerprint)) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.abbrev()) @@ -251,8 +252,8 @@ class WriteableSSKFileURI(_BaseURI): @implementer(IURI, IMutableFileURI) class ReadonlySSKFileURI(_BaseURI): - BASE_STRING='URI:SSK-RO:' - STRING_RE=re.compile('^URI:SSK-RO:'+BASE32STR_128bits+':'+BASE32STR_256bits+'$') + BASE_STRING=b'URI:SSK-RO:' + STRING_RE=re.compile(b'^URI:SSK-RO:'+BASE32STR_128bits+b':'+BASE32STR_256bits+b'$') def __init__(self, readkey, fingerprint): self.readkey = readkey @@ -270,8 +271,8 @@ class ReadonlySSKFileURI(_BaseURI): def to_string(self): assert isinstance(self.readkey, str) assert isinstance(self.fingerprint, str) - return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey), - base32.b2a(self.fingerprint)) + return b'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey), + base32.b2a(self.fingerprint)) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.abbrev()) @@ -298,8 +299,8 @@ class ReadonlySSKFileURI(_BaseURI): @implementer(IVerifierURI) class SSKVerifierURI(_BaseURI): - BASE_STRING='URI:SSK-Verifier:' - STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'$') + BASE_STRING=b'URI:SSK-Verifier:' + STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'$') def __init__(self, storage_index, fingerprint): assert len(storage_index) == 16 @@ -316,8 +317,8 @@ class SSKVerifierURI(_BaseURI): def to_string(self): assert isinstance(self.storage_index, str) assert isinstance(self.fingerprint, str) - return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), - base32.b2a(self.fingerprint)) + return b'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), + base32.b2a(self.fingerprint)) def is_readonly(self): return True @@ -335,8 +336,8 @@ class SSKVerifierURI(_BaseURI): @implementer(IURI, IMutableFileURI) class WriteableMDMFFileURI(_BaseURI): - BASE_STRING='URI:MDMF:' - STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)') + BASE_STRING=b'URI:MDMF:' + STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)') def __init__(self, writekey, fingerprint): self.writekey = writekey @@ -355,8 +356,8 @@ class WriteableMDMFFileURI(_BaseURI): def to_string(self): assert isinstance(self.writekey, str) assert isinstance(self.fingerprint, str) - ret = 'URI:MDMF:%s:%s' % (base32.b2a(self.writekey), - base32.b2a(self.fingerprint)) + ret = b'URI:MDMF:%s:%s' % (base32.b2a(self.writekey), + base32.b2a(self.fingerprint)) return ret def __repr__(self): @@ -384,8 +385,8 @@ class WriteableMDMFFileURI(_BaseURI): @implementer(IURI, IMutableFileURI) class ReadonlyMDMFFileURI(_BaseURI): - BASE_STRING='URI:MDMF-RO:' - STRING_RE=re.compile('^' +BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)') + BASE_STRING=b'URI:MDMF-RO:' + STRING_RE=re.compile(b'^' +BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)') def __init__(self, readkey, fingerprint): self.readkey = readkey @@ -404,8 +405,8 @@ class ReadonlyMDMFFileURI(_BaseURI): def to_string(self): assert isinstance(self.readkey, str) assert isinstance(self.fingerprint, str) - ret = 'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey), - base32.b2a(self.fingerprint)) + ret = b'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey), + base32.b2a(self.fingerprint)) return ret def __repr__(self): @@ -433,8 +434,8 @@ class ReadonlyMDMFFileURI(_BaseURI): @implementer(IVerifierURI) class MDMFVerifierURI(_BaseURI): - BASE_STRING='URI:MDMF-Verifier:' - STRING_RE=re.compile('^'+BASE_STRING+BASE32STR_128bits+':'+BASE32STR_256bits+'(:|$)') + BASE_STRING=b'URI:MDMF-Verifier:' + STRING_RE=re.compile(b'^'+BASE_STRING+BASE32STR_128bits+b':'+BASE32STR_256bits+b'(:|$)') def __init__(self, storage_index, fingerprint): assert len(storage_index) == 16 @@ -451,8 +452,8 @@ class MDMFVerifierURI(_BaseURI): def to_string(self): assert isinstance(self.storage_index, str) assert isinstance(self.fingerprint, str) - ret = 'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index), - base32.b2a(self.fingerprint)) + ret = b'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index), + base32.b2a(self.fingerprint)) return ret def is_readonly(self): @@ -494,12 +495,12 @@ class _DirectoryBaseURI(_BaseURI): return self.BASE_STRING+bits def abbrev(self): - return self._filenode_uri.to_string().split(':')[2][:5] + return self._filenode_uri.to_string().split(b':')[2][:5] def abbrev_si(self): si = self._filenode_uri.get_storage_index() if si is None: - return "" + return b"" return base32.b2a(si)[:5] def is_mutable(self): @@ -518,8 +519,8 @@ class _DirectoryBaseURI(_BaseURI): @implementer(IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): - BASE_STRING='URI:DIR2:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=WriteableSSKFileURI def __init__(self, filenode_uri=None): @@ -537,8 +538,8 @@ class DirectoryURI(_DirectoryBaseURI): @implementer(IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): - BASE_STRING='URI:DIR2-RO:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-RO:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=ReadonlySSKFileURI def __init__(self, filenode_uri=None): @@ -571,8 +572,8 @@ class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): class ImmutableDirectoryURI(_ImmutableDirectoryBaseURI): - BASE_STRING='URI:DIR2-CHK:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-CHK:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=CHKFileURI def get_verify_cap(self): @@ -581,8 +582,8 @@ class ImmutableDirectoryURI(_ImmutableDirectoryBaseURI): class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): - BASE_STRING='URI:DIR2-LIT:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-LIT:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=LiteralFileURI def get_verify_cap(self): @@ -593,8 +594,8 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): @implementer(IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): - BASE_STRING='URI:DIR2-MDMF:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-MDMF:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=WriteableMDMFFileURI def __init__(self, filenode_uri=None): @@ -615,8 +616,8 @@ class MDMFDirectoryURI(_DirectoryBaseURI): @implementer(IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): - BASE_STRING='URI:DIR2-MDMF-RO:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-MDMF-RO:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=ReadonlyMDMFFileURI def __init__(self, filenode_uri=None): @@ -653,8 +654,8 @@ def wrap_dirnode_cap(filecap): @implementer(IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): - BASE_STRING='URI:DIR2-MDMF-Verifier:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-MDMF-Verifier:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=MDMFVerifierURI def __init__(self, filenode_uri=None): @@ -678,8 +679,8 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): @implementer(IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): - BASE_STRING='URI:DIR2-Verifier:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-Verifier:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=SSKVerifierURI def __init__(self, filenode_uri=None): @@ -702,8 +703,8 @@ class DirectoryURIVerifier(_DirectoryBaseURI): @implementer(IVerifierURI) class ImmutableDirectoryURIVerifier(DirectoryURIVerifier): - BASE_STRING='URI:DIR2-CHK-Verifier:' - BASE_STRING_RE=re.compile('^'+BASE_STRING) + BASE_STRING=b'URI:DIR2-CHK-Verifier:' + BASE_STRING_RE=re.compile(b'^'+BASE_STRING) INNER_URI_CLASS=CHKFileVerifierURI @@ -725,12 +726,15 @@ class UnknownURI(object): return None -ALLEGED_READONLY_PREFIX = 'ro.' -ALLEGED_IMMUTABLE_PREFIX = 'imm.' +ALLEGED_READONLY_PREFIX = b'ro.' +ALLEGED_IMMUTABLE_PREFIX = b'imm.' def from_string(u, deep_immutable=False, name=u""): - if not isinstance(u, str): - raise TypeError("URI must be str: %r" % (u,)) + """Create URI from either unicode or byte string.""" + if isinstance(u, unicode): + u = u.encode("utf-8") + if not isinstance(u, bytes): + raise TypeError("URI must be unicode string or bytes: %r" % (u,)) # We allow and check ALLEGED_READONLY_PREFIX or ALLEGED_IMMUTABLE_PREFIX # on all URIs, even though we would only strictly need to do so for caps of @@ -748,62 +752,62 @@ def from_string(u, deep_immutable=False, name=u""): error = None try: - if s.startswith('URI:CHK:'): + if s.startswith(b'URI:CHK:'): return CHKFileURI.init_from_string(s) - elif s.startswith('URI:CHK-Verifier:'): + elif s.startswith(b'URI:CHK-Verifier:'): return CHKFileVerifierURI.init_from_string(s) - elif s.startswith('URI:LIT:'): + elif s.startswith(b'URI:LIT:'): return LiteralFileURI.init_from_string(s) - elif s.startswith('URI:SSK:'): + elif s.startswith(b'URI:SSK:'): if can_be_writeable: return WriteableSSKFileURI.init_from_string(s) kind = "URI:SSK file writecap" - elif s.startswith('URI:SSK-RO:'): + elif s.startswith(b'URI:SSK-RO:'): if can_be_mutable: return ReadonlySSKFileURI.init_from_string(s) kind = "URI:SSK-RO readcap to a mutable file" - elif s.startswith('URI:SSK-Verifier:'): + elif s.startswith(b'URI:SSK-Verifier:'): return SSKVerifierURI.init_from_string(s) - elif s.startswith('URI:MDMF:'): + elif s.startswith(b'URI:MDMF:'): if can_be_writeable: return WriteableMDMFFileURI.init_from_string(s) kind = "URI:MDMF file writecap" - elif s.startswith('URI:MDMF-RO:'): + elif s.startswith(b'URI:MDMF-RO:'): if can_be_mutable: return ReadonlyMDMFFileURI.init_from_string(s) kind = "URI:MDMF-RO readcap to a mutable file" - elif s.startswith('URI:MDMF-Verifier:'): + elif s.startswith(b'URI:MDMF-Verifier:'): return MDMFVerifierURI.init_from_string(s) - elif s.startswith('URI:DIR2:'): + elif s.startswith(b'URI:DIR2:'): if can_be_writeable: return DirectoryURI.init_from_string(s) kind = "URI:DIR2 directory writecap" - elif s.startswith('URI:DIR2-RO:'): + elif s.startswith(b'URI:DIR2-RO:'): if can_be_mutable: return ReadonlyDirectoryURI.init_from_string(s) kind = "URI:DIR2-RO readcap to a mutable directory" - elif s.startswith('URI:DIR2-Verifier:'): + elif s.startswith(b'URI:DIR2-Verifier:'): return DirectoryURIVerifier.init_from_string(s) - elif s.startswith('URI:DIR2-CHK:'): + elif s.startswith(b'URI:DIR2-CHK:'): return ImmutableDirectoryURI.init_from_string(s) - elif s.startswith('URI:DIR2-CHK-Verifier:'): + elif s.startswith(b'URI:DIR2-CHK-Verifier:'): return ImmutableDirectoryURIVerifier.init_from_string(s) - elif s.startswith('URI:DIR2-LIT:'): + elif s.startswith(b'URI:DIR2-LIT:'): return LiteralDirectoryURI.init_from_string(s) - elif s.startswith('URI:DIR2-MDMF:'): + elif s.startswith(b'URI:DIR2-MDMF:'): if can_be_writeable: return MDMFDirectoryURI.init_from_string(s) kind = "URI:DIR2-MDMF directory writecap" - elif s.startswith('URI:DIR2-MDMF-RO:'): + elif s.startswith(b'URI:DIR2-MDMF-RO:'): if can_be_mutable: return ReadonlyMDMFDirectoryURI.init_from_string(s) kind = "URI:DIR2-MDMF-RO readcap to a mutable directory" - elif s.startswith('URI:DIR2-MDMF-Verifier:'): + elif s.startswith(b'URI:DIR2-MDMF-Verifier:'): return MDMFDirectoryURIVerifier.init_from_string(s) - elif s.startswith('x-tahoe-future-test-writeable:') and not can_be_writeable: + elif s.startswith(b'x-tahoe-future-test-writeable:') and not can_be_writeable: # For testing how future writeable caps would behave in read-only contexts. kind = "x-tahoe-future-test-writeable: testing cap" - elif s.startswith('x-tahoe-future-test-mutable:') and not can_be_mutable: + elif s.startswith(b'x-tahoe-future-test-mutable:') and not can_be_mutable: # For testing how future mutable readcaps would behave in immutable contexts. kind = "x-tahoe-future-test-mutable: testing cap" else: @@ -829,18 +833,22 @@ def is_uri(s): return False def is_literal_file_uri(s): - if not isinstance(s, str): + if isinstance(s, unicode): + s = s.encode("utf-8") + if not isinstance(s, bytes): return False - return (s.startswith('URI:LIT:') or - s.startswith(ALLEGED_READONLY_PREFIX + 'URI:LIT:') or - s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:LIT:')) + return (s.startswith(b'URI:LIT:') or + s.startswith(ALLEGED_READONLY_PREFIX + b'URI:LIT:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:LIT:')) def has_uri_prefix(s): - if not isinstance(s, str): + if isinstance(s, unicode): + s = s.encode("utf-8") + if not isinstance(s, bytes): return False - return (s.startswith("URI:") or - s.startswith(ALLEGED_READONLY_PREFIX + 'URI:') or - s.startswith(ALLEGED_IMMUTABLE_PREFIX + 'URI:')) + return (s.startswith(b"URI:") or + s.startswith(ALLEGED_READONLY_PREFIX + b'URI:') or + s.startswith(ALLEGED_IMMUTABLE_PREFIX + b'URI:')) # These take the same keyword arguments as from_string above. @@ -850,26 +858,26 @@ def from_string_dirnode(s, **kwargs): _assert(IDirnodeURI.providedBy(u)) return u -registerAdapter(from_string_dirnode, str, IDirnodeURI) +registerAdapter(from_string_dirnode, bytes, IDirnodeURI) def from_string_filenode(s, **kwargs): u = from_string(s, **kwargs) _assert(IFileURI.providedBy(u)) return u -registerAdapter(from_string_filenode, str, IFileURI) +registerAdapter(from_string_filenode, bytes, IFileURI) def from_string_mutable_filenode(s, **kwargs): u = from_string(s, **kwargs) _assert(IMutableFileURI.providedBy(u)) return u -registerAdapter(from_string_mutable_filenode, str, IMutableFileURI) +registerAdapter(from_string_mutable_filenode, bytes, IMutableFileURI) def from_string_verifier(s, **kwargs): u = from_string(s, **kwargs) _assert(IVerifierURI.providedBy(u)) return u -registerAdapter(from_string_verifier, str, IVerifierURI) +registerAdapter(from_string_verifier, bytes, IVerifierURI) def pack_extension(data): @@ -877,30 +885,32 @@ def pack_extension(data): for k in sorted(data.keys()): value = data[k] if isinstance(value, (int, long)): - value = "%d" % value - assert isinstance(value, str), k - assert re.match(r'^[a-zA-Z_\-]+$', k) - pieces.append(k + ':' + hashutil.netstring(value)) - uri_extension = ''.join(pieces) + value = b"%d" % value + if isinstance(k, unicode): + k = k.encode("utf-8") + assert isinstance(value, bytes), k + assert re.match(br'^[a-zA-Z_\-]+$', k) + pieces.append(k + b':' + hashutil.netstring(value)) + uri_extension = b''.join(pieces) return uri_extension def unpack_extension(data): d = {} while data: - colon = data.index(':') + colon = data.index(b':') key = data[:colon] data = data[colon+1:] - colon = data.index(':') + colon = data.index(b':') number = data[:colon] length = int(number) data = data[colon+1:] value = data[:length] - assert data[length] == ',' + assert data[length] == b',' data = data[length+1:] - d[key] = value + d[key.decode("utf-8")] = value # convert certain things to numbers for intkey in ('size', 'segment_size', 'num_segments', @@ -914,6 +924,7 @@ def unpack_extension_readable(data): unpacked = unpack_extension(data) unpacked["UEB_hash"] = hashutil.uri_extension_hash(data) for k in sorted(unpacked.keys()): + k = k.decode("utf-8") if 'hash' in k: unpacked[k] = base32.b2a(unpacked[k]) return unpacked From 0e15712e347c75ea843629ed6b53afb210d8b2d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Aug 2020 13:53:27 -0400 Subject: [PATCH 6/7] Tests pass on Python 2 and 3. --- src/allmydata/test/test_uri.py | 26 +++++++++++------------ src/allmydata/uri.py | 39 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index 4b9c229f0..005cdf597 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -202,24 +202,24 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): class Extension(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - data = {"stuff": b"value", - "size": 12, - "needed_shares": 3, - "big_hash": hashutil.tagged_hash(b"foo", b"bar"), + data = {b"stuff": b"value", + b"size": 12, + b"needed_shares": 3, + b"big_hash": hashutil.tagged_hash(b"foo", b"bar"), } ext = uri.pack_extension(data) d = uri.unpack_extension(ext) - self.failUnlessReallyEqual(d["stuff"], b"value") - self.failUnlessReallyEqual(d["size"], 12) - self.failUnlessReallyEqual(d["big_hash"], hashutil.tagged_hash(b"foo", b"bar")) + self.failUnlessReallyEqual(d[b"stuff"], b"value") + self.failUnlessReallyEqual(d[b"size"], 12) + self.failUnlessReallyEqual(d[b"big_hash"], hashutil.tagged_hash(b"foo", b"bar")) readable = uri.unpack_extension_readable(ext) - self.failUnlessReallyEqual(readable["needed_shares"], 3) - self.failUnlessReallyEqual(readable["stuff"], b"value") - self.failUnlessReallyEqual(readable["size"], 12) - self.failUnlessReallyEqual(readable["big_hash"], + self.failUnlessReallyEqual(readable[b"needed_shares"], 3) + self.failUnlessReallyEqual(readable[b"stuff"], b"value") + self.failUnlessReallyEqual(readable[b"size"], 12) + self.failUnlessReallyEqual(readable[b"big_hash"], base32.b2a(hashutil.tagged_hash(b"foo", b"bar"))) - self.failUnlessReallyEqual(readable["UEB_hash"], + self.failUnlessReallyEqual(readable[b"UEB_hash"], base32.b2a(hashutil.uri_extension_hash(ext))) class Unknown(testutil.ReallyEqualMixin, unittest.TestCase): @@ -254,7 +254,7 @@ class Unknown(testutil.ReallyEqualMixin, unittest.TestCase): class Constraint(testutil.ReallyEqualMixin, unittest.TestCase): def test_constraint(self): - bad = "http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/" + bad = b"http://127.0.0.1:3456/uri/URI%3ADIR2%3Agh3l5rbvnv2333mrfvalmjfr4i%3Alz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma/" self.failUnlessRaises(uri.BadURIError, uri.DirectoryURI.init_from_string, bad) fileURI = b'URI:CHK:gh3l5rbvnv2333mrfvalmjfr4i:lz6l7u3z3b7g37s4zkdmfpx5ly4ib4m6thrpbusi6ys62qtc6mma:3:10:345834' uri.CHKFileURI.init_from_string(fileURI) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 70fd80c92..62f4025fb 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -167,7 +167,7 @@ class LiteralFileURI(_BaseURI): def __init__(self, data=None): if data is not None: - assert isinstance(data, str) + assert isinstance(data, bytes) self.data = data @classmethod @@ -222,8 +222,8 @@ class WriteableSSKFileURI(_BaseURI): return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.writekey, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.writekey, bytes) + assert isinstance(self.fingerprint, bytes) return b'URI:SSK:%s:%s' % (base32.b2a(self.writekey), base32.b2a(self.fingerprint)) @@ -269,8 +269,8 @@ class ReadonlySSKFileURI(_BaseURI): return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.readkey, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.readkey, bytes) + assert isinstance(self.fingerprint, bytes) return b'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey), base32.b2a(self.fingerprint)) @@ -315,8 +315,8 @@ class SSKVerifierURI(_BaseURI): return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.storage_index, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.storage_index, bytes) + assert isinstance(self.fingerprint, bytes) return b'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), base32.b2a(self.fingerprint)) @@ -354,8 +354,8 @@ class WriteableMDMFFileURI(_BaseURI): return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.writekey, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.writekey, bytes) + assert isinstance(self.fingerprint, bytes) ret = b'URI:MDMF:%s:%s' % (base32.b2a(self.writekey), base32.b2a(self.fingerprint)) return ret @@ -403,8 +403,8 @@ class ReadonlyMDMFFileURI(_BaseURI): return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.readkey, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.readkey, bytes) + assert isinstance(self.fingerprint, bytes) ret = b'URI:MDMF-RO:%s:%s' % (base32.b2a(self.readkey), base32.b2a(self.fingerprint)) return ret @@ -450,8 +450,8 @@ class MDMFVerifierURI(_BaseURI): return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) def to_string(self): - assert isinstance(self.storage_index, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.storage_index, bytes) + assert isinstance(self.fingerprint, bytes) ret = b'URI:MDMF-Verifier:%s:%s' % (si_b2a(self.storage_index), base32.b2a(self.fingerprint)) return ret @@ -907,14 +907,14 @@ def unpack_extension(data): data = data[colon+1:] value = data[:length] - assert data[length] == b',' + assert data[length:length+1] == b',' data = data[length+1:] - d[key.decode("utf-8")] = value + d[key] = value # convert certain things to numbers - for intkey in ('size', 'segment_size', 'num_segments', - 'needed_shares', 'total_shares'): + for intkey in (b'size', b'segment_size', b'num_segments', + b'needed_shares', b'total_shares'): if intkey in d: d[intkey] = int(d[intkey]) return d @@ -922,10 +922,9 @@ def unpack_extension(data): def unpack_extension_readable(data): unpacked = unpack_extension(data) - unpacked["UEB_hash"] = hashutil.uri_extension_hash(data) + unpacked[b"UEB_hash"] = hashutil.uri_extension_hash(data) for k in sorted(unpacked.keys()): - k = k.decode("utf-8") - if 'hash' in k: + if b'hash' in k: unpacked[k] = base32.b2a(unpacked[k]) return unpacked From 2cc21e98937968dadddca6008102805f7c24bdd1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 24 Aug 2020 13:57:46 -0400 Subject: [PATCH 7/7] Finish the port. --- newsfragments/3367.minor | 0 src/allmydata/test/test_uri.py | 5 +++-- src/allmydata/uri.py | 18 ++++++++++++++++++ src/allmydata/util/_python3.py | 3 ++- 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3367.minor diff --git a/newsfragments/3367.minor b/newsfragments/3367.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index 005cdf597..f89fae151 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -1,5 +1,6 @@ - """ +Tests for allmydata.uri. + Ported to Python 3. """ @@ -10,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 import os from twisted.trial import unittest diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 62f4025fb..b601226da 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -1,3 +1,21 @@ +""" +URIs (kinda sorta, really they're capabilities?). + +Ported to Python 3. + +Methods ending in to_string() are actually to_bytes(), possibly should be fixed +in follow-up port. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Don't import bytes, to prevent leaks. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 + from past.builtins import unicode, long import re diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 84c08a36c..2351f3707 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -20,7 +20,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ @@ -34,6 +34,7 @@ PORTED_MODULES = [ "allmydata.immutable.happiness_upload", "allmydata.storage.crawler", "allmydata.test.common_py3", + "allmydata.uri", "allmydata.util._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil",