diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 36eb55bfb..e6aebab17 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,4 +1,8 @@ +from future.utils import PY2 +if not PY2: + long = int + from zope.interface import Interface, Attribute from twisted.plugin import ( IPlugin, diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 008619d64..14139d81e 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -1,6 +1,9 @@ import os, time, struct -import cPickle as pickle +try: + import cPickle as pickle +except ImportError: + import pickle from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 261d39af3..2cd35bdd4 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,5 +1,7 @@ from __future__ import print_function +from future.utils import PY2 + import os from random import randrange from six.moves import StringIO @@ -11,7 +13,8 @@ from twisted.trial import unittest from ..util.assertutil import precondition from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, get_io_encoding) -from ..scripts import runner +if PY2: + from ..scripts import runner # hack for tests to run from .common_py3 import SignalMixin diff --git a/src/allmydata/test/test_uri.py b/src/allmydata/test/test_uri.py index de9f42a1e..cead46ea0 100644 --- a/src/allmydata/test/test_uri.py +++ b/src/allmydata/test/test_uri.py @@ -3,10 +3,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import str -from builtins import * +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + import os from twisted.trial import unittest from allmydata import uri @@ -97,7 +97,7 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase): class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 @@ -163,7 +163,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): self.failIf(v3.is_mutable()) def test_pack_badly(self): - key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" storage_index = hashutil.storage_index_hash(key) uri_extension_hash = hashutil.uri_extension_hash("stuff") needed_shares = 25 @@ -251,8 +251,8 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase): class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): def setUp(self): - self.writekey = "\x01" * 16 - self.fingerprint = "\x02" * 32 + self.writekey = b"\x01" * 16 + self.fingerprint = b"\x02" * 32 self.readkey = hashutil.ssk_readkey_hash(self.writekey) self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) @@ -476,8 +476,8 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): def test_pack(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 n = uri.WriteableSSKFileURI(writekey, fingerprint) u1 = uri.DirectoryURI(n) @@ -619,8 +619,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(u1.abbrev_si(), "") def test_mdmf(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) self.failIf(d1.is_readonly()) @@ -643,8 +643,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessIsInstance(d3, uri.UnknownURI) def test_mdmf_attenuation(self): - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) @@ -684,8 +684,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): def test_mdmf_verifier(self): # I'm not sure what I want to write here yet. - writekey = "\x01" * 16 - fingerprint = "\x02" * 32 + writekey = b"\x01" * 16 + fingerprint = b"\x02" * 32 uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) d1 = uri.MDMFDirectoryURI(uri1) v1 = d1.get_verify_cap() diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1e9dd0f49..05e3b2ac8 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,5 +1,9 @@ from __future__ import print_function +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + import six import os, time, sys import yaml diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 030d24384..3c8c83255 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -3,10 +3,10 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() -from builtins import * -from builtins import object +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + import re from zope.interface import implementer @@ -32,8 +32,8 @@ class BadURIError(CapConstraintError): # - make variable and method names consistently use _uri for an URI string, # and _cap for a Cap object (decoded URI) -BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) -BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) +BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR.decode('utf8'), base32.BASE32CHAR_3bits.decode('utf8')) +BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR.decode('utf8'), base32.BASE32CHAR_1bits.decode('utf8')) NUMBER='([0-9]+)' @@ -82,7 +82,8 @@ class CHKFileURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)), + return cls(base32.a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8')), int(mo.group(3)), int(mo.group(4)), int(mo.group(5))) def to_string(self): @@ -138,7 +139,8 @@ class CHKFileVerifierURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)), + return cls(si_a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8')), int(mo.group(3)), int(mo.group(4)), int(mo.group(5))) def to_string(self): @@ -170,7 +172,7 @@ class CHKFileVerifierURI(_BaseURI): class LiteralFileURI(_BaseURI): BASE_STRING='URI:LIT:' - STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$') + STRING_RE=re.compile('^URI:LIT:'+str(base32.BASE32STR_anybytes)+'$') def __init__(self, data=None): if data is not None: @@ -182,7 +184,7 @@ class LiteralFileURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1))) + return cls(base32.a2b(bytes(mo.group(1), 'utf8'))) def to_string(self): return 'URI:LIT:%s' % base32.b2a(self.data) @@ -215,6 +217,7 @@ class WriteableSSKFileURI(_BaseURI): BASE32STR_256bits+'$') def __init__(self, writekey, fingerprint): + assert isinstance(writekey, bytes) self.writekey = writekey self.readkey = hashutil.ssk_readkey_hash(writekey) self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) @@ -226,13 +229,14 @@ class WriteableSSKFileURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(base32.a2b(mo.group(1).encode('utf8')), + base32.a2b(mo.group(2).encode('utf8'))) def to_string(self): - assert isinstance(self.writekey, str) - assert isinstance(self.fingerprint, str) - return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey), - base32.b2a(self.fingerprint)) + assert isinstance(self.writekey, bytes) + assert isinstance(self.fingerprint, bytes) + return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey).decode('utf8'), + base32.b2a(self.fingerprint).decode('utf8')) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.abbrev()) @@ -273,13 +277,14 @@ class ReadonlySSKFileURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(base32.a2b(mo.group(1).encode('utf8')), + base32.a2b(mo.group(2).encode('utf8'))) def to_string(self): - assert isinstance(self.readkey, str) - assert isinstance(self.fingerprint, str) - return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey), - base32.b2a(self.fingerprint)) + assert isinstance(self.readkey, bytes) + assert isinstance(self.fingerprint, bytes) + return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey).decode('utf8'), + base32.b2a(self.fingerprint).decode('utf8')) def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.abbrev()) @@ -319,11 +324,13 @@ class SSKVerifierURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(si_a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8'))) def to_string(self): - assert isinstance(self.storage_index, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.storage_index, bytes) + assert isinstance(self.fingerprint, bytes) + XXX return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), base32.b2a(self.fingerprint)) @@ -358,11 +365,12 @@ class WriteableMDMFFileURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(base32.a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8'))) def to_string(self): - assert isinstance(self.writekey, str) - assert isinstance(self.fingerprint, str) + assert isinstance(self.writekey, bytes) + assert isinstance(self.fingerprint, bytes) ret = 'URI:MDMF:%s:%s' % (base32.b2a(self.writekey), base32.b2a(self.fingerprint)) return ret @@ -407,7 +415,8 @@ class ReadonlyMDMFFileURI(_BaseURI): if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(base32.a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8'))) def to_string(self): assert isinstance(self.readkey, str) @@ -454,7 +463,8 @@ class MDMFVerifierURI(_BaseURI): mo = cls.STRING_RE.search(uri) if not mo: raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) - return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) + return cls(si_a2b(bytes(mo.group(1), 'utf8')), + base32.a2b(bytes(mo.group(2), 'utf8'))) def to_string(self): assert isinstance(self.storage_index, str)