an attempt to have the encoding/decoding dance done properly for one case

This commit is contained in:
Maciej Fijalkowski 2020-08-04 17:25:40 +02:00
parent 9f77bf7d60
commit 82409138b7
6 changed files with 70 additions and 46 deletions

View File

@ -1,4 +1,8 @@
from future.utils import PY2
if not PY2:
long = int
from zope.interface import Interface, Attribute from zope.interface import Interface, Attribute
from twisted.plugin import ( from twisted.plugin import (
IPlugin, IPlugin,

View File

@ -1,6 +1,9 @@
import os, time, struct import os, time, struct
import cPickle as pickle try:
import cPickle as pickle
except ImportError:
import pickle
from twisted.internet import reactor from twisted.internet import reactor
from twisted.application import service from twisted.application import service
from allmydata.storage.common import si_b2a from allmydata.storage.common import si_b2a

View File

@ -1,5 +1,7 @@
from __future__ import print_function from __future__ import print_function
from future.utils import PY2
import os import os
from random import randrange from random import randrange
from six.moves import StringIO from six.moves import StringIO
@ -11,7 +13,8 @@ from twisted.trial import unittest
from ..util.assertutil import precondition from ..util.assertutil import precondition
from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding, from allmydata.util.encodingutil import (unicode_platform, get_filesystem_encoding,
get_io_encoding) get_io_encoding)
from ..scripts import runner if PY2:
from ..scripts import runner # hack for tests to run
from .common_py3 import SignalMixin from .common_py3 import SignalMixin

View File

@ -3,10 +3,10 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
from future import standard_library from future.utils import PY2
standard_library.install_aliases() if PY2:
from builtins import str from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401
from builtins import *
import os import os
from twisted.trial import unittest from twisted.trial import unittest
from allmydata import uri from allmydata import uri
@ -97,7 +97,7 @@ class Compare(testutil.ReallyEqualMixin, unittest.TestCase):
class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase): class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
def test_pack(self): def test_pack(self):
key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
storage_index = hashutil.storage_index_hash(key) storage_index = hashutil.storage_index_hash(key)
uri_extension_hash = hashutil.uri_extension_hash("stuff") uri_extension_hash = hashutil.uri_extension_hash("stuff")
needed_shares = 25 needed_shares = 25
@ -163,7 +163,7 @@ class CHKFile(testutil.ReallyEqualMixin, unittest.TestCase):
self.failIf(v3.is_mutable()) self.failIf(v3.is_mutable())
def test_pack_badly(self): def test_pack_badly(self):
key = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
storage_index = hashutil.storage_index_hash(key) storage_index = hashutil.storage_index_hash(key)
uri_extension_hash = hashutil.uri_extension_hash("stuff") uri_extension_hash = hashutil.uri_extension_hash("stuff")
needed_shares = 25 needed_shares = 25
@ -251,8 +251,8 @@ class Constraint(testutil.ReallyEqualMixin, unittest.TestCase):
class Mutable(testutil.ReallyEqualMixin, unittest.TestCase): class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
def setUp(self): def setUp(self):
self.writekey = "\x01" * 16 self.writekey = b"\x01" * 16
self.fingerprint = "\x02" * 32 self.fingerprint = b"\x02" * 32
self.readkey = hashutil.ssk_readkey_hash(self.writekey) self.readkey = hashutil.ssk_readkey_hash(self.writekey)
self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
@ -476,8 +476,8 @@ class Mutable(testutil.ReallyEqualMixin, unittest.TestCase):
class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase): class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
def test_pack(self): def test_pack(self):
writekey = "\x01" * 16 writekey = b"\x01" * 16
fingerprint = "\x02" * 32 fingerprint = b"\x02" * 32
n = uri.WriteableSSKFileURI(writekey, fingerprint) n = uri.WriteableSSKFileURI(writekey, fingerprint)
u1 = uri.DirectoryURI(n) u1 = uri.DirectoryURI(n)
@ -619,8 +619,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
self.failUnlessReallyEqual(u1.abbrev_si(), "<LIT>") self.failUnlessReallyEqual(u1.abbrev_si(), "<LIT>")
def test_mdmf(self): def test_mdmf(self):
writekey = "\x01" * 16 writekey = b"\x01" * 16
fingerprint = "\x02" * 32 fingerprint = b"\x02" * 32
uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
d1 = uri.MDMFDirectoryURI(uri1) d1 = uri.MDMFDirectoryURI(uri1)
self.failIf(d1.is_readonly()) self.failIf(d1.is_readonly())
@ -643,8 +643,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
self.failUnlessIsInstance(d3, uri.UnknownURI) self.failUnlessIsInstance(d3, uri.UnknownURI)
def test_mdmf_attenuation(self): def test_mdmf_attenuation(self):
writekey = "\x01" * 16 writekey = b"\x01" * 16
fingerprint = "\x02" * 32 fingerprint = b"\x02" * 32
uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
d1 = uri.MDMFDirectoryURI(uri1) d1 = uri.MDMFDirectoryURI(uri1)
@ -684,8 +684,8 @@ class Dirnode(testutil.ReallyEqualMixin, unittest.TestCase):
def test_mdmf_verifier(self): def test_mdmf_verifier(self):
# I'm not sure what I want to write here yet. # I'm not sure what I want to write here yet.
writekey = "\x01" * 16 writekey = b"\x01" * 16
fingerprint = "\x02" * 32 fingerprint = b"\x02" * 32
uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint) uri1 = uri.WriteableMDMFFileURI(writekey, fingerprint)
d1 = uri.MDMFDirectoryURI(uri1) d1 = uri.MDMFDirectoryURI(uri1)
v1 = d1.get_verify_cap() v1 = d1.get_verify_cap()

View File

@ -1,5 +1,9 @@
from __future__ import print_function from __future__ import print_function
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401
import six import six
import os, time, sys import os, time, sys
import yaml import yaml

View File

@ -3,10 +3,10 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals from __future__ import unicode_literals
from future import standard_library from future.utils import PY2
standard_library.install_aliases() if PY2:
from builtins import * from builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401
from builtins import object
import re import re
from zope.interface import implementer from zope.interface import implementer
@ -32,8 +32,8 @@ class BadURIError(CapConstraintError):
# - make variable and method names consistently use _uri for an URI string, # - make variable and method names consistently use _uri for an URI string,
# and _cap for a Cap object (decoded URI) # and _cap for a Cap object (decoded URI)
BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_3bits) BASE32STR_128bits = '(%s{25}%s)' % (base32.BASE32CHAR.decode('utf8'), base32.BASE32CHAR_3bits.decode('utf8'))
BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR, base32.BASE32CHAR_1bits) BASE32STR_256bits = '(%s{51}%s)' % (base32.BASE32CHAR.decode('utf8'), base32.BASE32CHAR_1bits.decode('utf8'))
NUMBER='([0-9]+)' NUMBER='([0-9]+)'
@ -82,7 +82,8 @@ class CHKFileURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2)), return cls(base32.a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')),
int(mo.group(3)), int(mo.group(4)), int(mo.group(5))) int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
def to_string(self): def to_string(self):
@ -138,7 +139,8 @@ class CHKFileVerifierURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2)), return cls(si_a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')),
int(mo.group(3)), int(mo.group(4)), int(mo.group(5))) int(mo.group(3)), int(mo.group(4)), int(mo.group(5)))
def to_string(self): def to_string(self):
@ -170,7 +172,7 @@ class CHKFileVerifierURI(_BaseURI):
class LiteralFileURI(_BaseURI): class LiteralFileURI(_BaseURI):
BASE_STRING='URI:LIT:' BASE_STRING='URI:LIT:'
STRING_RE=re.compile('^URI:LIT:'+base32.BASE32STR_anybytes+'$') STRING_RE=re.compile('^URI:LIT:'+str(base32.BASE32STR_anybytes)+'$')
def __init__(self, data=None): def __init__(self, data=None):
if data is not None: if data is not None:
@ -182,7 +184,7 @@ class LiteralFileURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1))) return cls(base32.a2b(bytes(mo.group(1), 'utf8')))
def to_string(self): def to_string(self):
return 'URI:LIT:%s' % base32.b2a(self.data) return 'URI:LIT:%s' % base32.b2a(self.data)
@ -215,6 +217,7 @@ class WriteableSSKFileURI(_BaseURI):
BASE32STR_256bits+'$') BASE32STR_256bits+'$')
def __init__(self, writekey, fingerprint): def __init__(self, writekey, fingerprint):
assert isinstance(writekey, bytes)
self.writekey = writekey self.writekey = writekey
self.readkey = hashutil.ssk_readkey_hash(writekey) self.readkey = hashutil.ssk_readkey_hash(writekey)
self.storage_index = hashutil.ssk_storage_index_hash(self.readkey) self.storage_index = hashutil.ssk_storage_index_hash(self.readkey)
@ -226,13 +229,14 @@ class WriteableSSKFileURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(base32.a2b(mo.group(1).encode('utf8')),
base32.a2b(mo.group(2).encode('utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.writekey, str) assert isinstance(self.writekey, bytes)
assert isinstance(self.fingerprint, str) assert isinstance(self.fingerprint, bytes)
return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey), return 'URI:SSK:%s:%s' % (base32.b2a(self.writekey).decode('utf8'),
base32.b2a(self.fingerprint)) base32.b2a(self.fingerprint).decode('utf8'))
def __repr__(self): def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.abbrev()) return "<%s %s>" % (self.__class__.__name__, self.abbrev())
@ -273,13 +277,14 @@ class ReadonlySSKFileURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(base32.a2b(mo.group(1).encode('utf8')),
base32.a2b(mo.group(2).encode('utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.readkey, str) assert isinstance(self.readkey, bytes)
assert isinstance(self.fingerprint, str) assert isinstance(self.fingerprint, bytes)
return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey), return 'URI:SSK-RO:%s:%s' % (base32.b2a(self.readkey).decode('utf8'),
base32.b2a(self.fingerprint)) base32.b2a(self.fingerprint).decode('utf8'))
def __repr__(self): def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.abbrev()) return "<%s %s>" % (self.__class__.__name__, self.abbrev())
@ -319,11 +324,13 @@ class SSKVerifierURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(si_a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.storage_index, str) assert isinstance(self.storage_index, bytes)
assert isinstance(self.fingerprint, str) assert isinstance(self.fingerprint, bytes)
XXX
return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index), return 'URI:SSK-Verifier:%s:%s' % (si_b2a(self.storage_index),
base32.b2a(self.fingerprint)) base32.b2a(self.fingerprint))
@ -358,11 +365,12 @@ class WriteableMDMFFileURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(base32.a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.writekey, str) assert isinstance(self.writekey, bytes)
assert isinstance(self.fingerprint, str) assert isinstance(self.fingerprint, bytes)
ret = 'URI:MDMF:%s:%s' % (base32.b2a(self.writekey), ret = 'URI:MDMF:%s:%s' % (base32.b2a(self.writekey),
base32.b2a(self.fingerprint)) base32.b2a(self.fingerprint))
return ret return ret
@ -407,7 +415,8 @@ class ReadonlyMDMFFileURI(_BaseURI):
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(base32.a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(base32.a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.readkey, str) assert isinstance(self.readkey, str)
@ -454,7 +463,8 @@ class MDMFVerifierURI(_BaseURI):
mo = cls.STRING_RE.search(uri) mo = cls.STRING_RE.search(uri)
if not mo: if not mo:
raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls)) raise BadURIError("'%s' doesn't look like a %s cap" % (uri, cls))
return cls(si_a2b(mo.group(1)), base32.a2b(mo.group(2))) return cls(si_a2b(bytes(mo.group(1), 'utf8')),
base32.a2b(bytes(mo.group(2), 'utf8')))
def to_string(self): def to_string(self):
assert isinstance(self.storage_index, str) assert isinstance(self.storage_index, str)