ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 13:07:56 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

uri: implement URI-processing classes, IFileURI/IDirnodeURI, use internally

Browse Source
This commit is contained in:
Brian Warner 2007-07-21 15:40:36 -07:00
parent 32fcf0b405
commit 1d9a58977f
14 changed files with 519 additions and 304 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
src/allmydata/dirnode.py
View File

@ -5,7 +5,8 @@ from twisted.application import service
from twisted.internet import defer
from foolscap import Referenceable
from allmydata import uri
from allmydata.interfaces import RIVirtualDriveServer, IDirectoryNode, IFileNode
from allmydata.interfaces import RIVirtualDriveServer, \
IDirectoryNode, IFileNode, IFileURI, IDirnodeURI, IURI
from allmydata.util import bencode, idlib, hashutil, fileutil
from allmydata.Crypto.Cipher import AES
@ -48,7 +49,8 @@ class VirtualDriveServer(service.MultiService, Referenceable):
def get_public_root_uri(self):
if self._root:
return uri.pack_dirnode_uri(self._myfurl, self._root)
u = uri.DirnodeURI(self._myfurl, self._root)
return u.to_string()
raise NoPublicRootError
remote_get_public_root_uri = get_public_root_uri
@ -121,16 +123,14 @@ class NotMutableError(Exception):
def create_directory_node(client, diruri):
assert uri.is_dirnode_uri(diruri)
if uri.is_mutable_dirnode_uri(diruri):
dirnode_class = MutableDirectoryNode
else:
dirnode_class = ImmutableDirectoryNode
(furl, key) = uri.unpack_dirnode_uri(diruri)
d = client.tub.getReference(furl)
u = IURI(diruri)
assert IDirnodeURI.providedBy(u)
d = client.tub.getReference(u.furl)
def _got(rref):
dirnode = dirnode_class(diruri, client, rref, key)
return dirnode
if isinstance(u, uri.DirnodeURI):
return MutableDirectoryNode(u, client, rref)
else: # uri.ReadOnlyDirnodeURI
return ImmutableDirectoryNode(u, client, rref)
d.addCallback(_got)
return d
@ -163,12 +163,14 @@ def decrypt(key, data):
class ImmutableDirectoryNode:
implements(IDirectoryNode)
def __init__(self, myuri, client, rref, readkey):
self._uri = myuri
def __init__(self, myuri, client, rref):
u = IDirnodeURI(myuri)
assert u.is_readonly()
self._uri = u.to_string()
self._client = client
self._tub = client.tub
self._rref = rref
self._readkey = readkey
self._readkey = u.readkey
self._writekey = None
self._write_enabler = None
self._index = hashutil.dir_index_hash(self._readkey)
@ -188,10 +190,7 @@ class ImmutableDirectoryNode:
def get_immutable_uri(self):
# return the dirnode URI for a read-only form of this directory
if self._mutable:
return uri.make_immutable_dirnode_uri(self._uri)
else:
return self._uri
return IDirnodeURI(self._uri).get_readonly().to_string()
def __hash__(self):
return hash((self.__class__, self._uri))
@ -256,7 +255,9 @@ class ImmutableDirectoryNode:
E_name = self._encrypt(self._readkey, name)
E_write = ""
if self._writekey and write_child:
assert isinstance(write_child, str)
E_write = self._encrypt(self._writekey, write_child)
assert isinstance(read_child, str)
E_read = self._encrypt(self._readkey, read_child)
d = self._rref.callRemote("set", self._index, self._write_enabler,
H_name, E_name, E_write, E_read)
@ -280,30 +281,28 @@ class ImmutableDirectoryNode:
return d
def _create_node(self, child_uri):
if uri.is_dirnode_uri(child_uri):
return create_directory_node(self._client, child_uri)
u = IURI(child_uri)
if IDirnodeURI.providedBy(u):
return create_directory_node(self._client, u)
else:
return defer.succeed(FileNode(child_uri, self._client))
return defer.succeed(FileNode(u, self._client))
def _split_uri(self, child_uri):
if uri.is_dirnode_uri(child_uri):
if uri.is_mutable_dirnode_uri(child_uri):
write = child_uri
read = uri.make_immutable_dirnode_uri(child_uri)
else:
write = None
read = child_uri
return (write, read)
return (None, child_uri) # file
u = IURI(child_uri)
if u.is_mutable() and not u.is_readonly():
write = u.to_string()
else:
write = None
read = u.get_readonly().to_string()
return (write, read)
def create_empty_directory(self, name):
if not self._mutable:
return defer.fail(NotMutableError())
child_writekey = hashutil.random_key()
my_furl, parent_writekey = uri.unpack_dirnode_uri(self._uri)
child_uri = uri.pack_dirnode_uri(my_furl, child_writekey)
child = MutableDirectoryNode(child_uri, self._client, self._rref,
child_writekey)
furl = IDirnodeURI(self._uri).furl
u = uri.DirnodeURI(furl, child_writekey)
child = MutableDirectoryNode(u, self._client, self._rref)
d = self._rref.callRemote("create_directory",
child._index, child._write_enabler)
d.addCallback(lambda index: self.set_node(name, child))
@ -362,8 +361,8 @@ class ImmutableDirectoryNode:
return d
def get_refresh_capability(self):
ro_uri = self.get_immutable_uri()
furl, rk = uri.unpack_dirnode_uri(ro_uri)
u = IDirnodeURI(self._uri).get_readonly()
rk = u.readkey
wk, we, rk, index = hashutil.generate_dirnode_keys_from_readkey(rk)
return "DIR-REFRESH:%s" % idlib.b2a(index)
@ -384,21 +383,28 @@ class ImmutableDirectoryNode:
class MutableDirectoryNode(ImmutableDirectoryNode):
implements(IDirectoryNode)
def __init__(self, myuri, client, rref, writekey):
readkey = hashutil.dir_read_key_hash(writekey)
ImmutableDirectoryNode.__init__(self, myuri, client, rref, readkey)
self._writekey = writekey
self._write_enabler = hashutil.dir_write_enabler_hash(writekey)
def __init__(self, myuri, client, rref):
u = IDirnodeURI(myuri)
assert not u.is_readonly()
self._writekey = u.writekey
self._write_enabler = hashutil.dir_write_enabler_hash(u.writekey)
readkey = hashutil.dir_read_key_hash(u.writekey)
self._uri = u.to_string()
self._client = client
self._tub = client.tub
self._rref = rref
self._readkey = readkey
self._index = hashutil.dir_index_hash(self._readkey)
self._mutable = True
def create_directory(client, furl):
write_key = hashutil.random_key()
(wk, we, rk, index) = \
hashutil.generate_dirnode_keys_from_writekey(write_key)
myuri = uri.pack_dirnode_uri(furl, wk)
u = uri.DirnodeURI(furl, wk)
d = client.tub.getReference(furl)
def _got_vdrive_server(vdrive_server):
node = MutableDirectoryNode(myuri, client, vdrive_server, wk)
node = MutableDirectoryNode(u, client, vdrive_server)
d2 = vdrive_server.callRemote("create_directory", index, we)
d2.addCallback(lambda res: node)
return d2
@ -409,12 +415,16 @@ class FileNode:
implements(IFileNode)
def __init__(self, uri, client):
self.uri = uri
u = IFileURI(uri)
self.uri = u.to_string()
self._client = client
def get_uri(self):
return self.uri
def get_size(self):
return IFileURI(self.uri).get_size()
def __hash__(self):
return hash((self.__class__, self.uri))
def __cmp__(self, them):
@ -425,10 +435,9 @@ class FileNode:
return cmp(self.uri, them.uri)
def get_refresh_capability(self):
t = uri.get_uri_type(self.uri)
if t == "CHK":
d = uri.unpack_uri(self.uri)
return "CHK-REFRESH:%s" % idlib.b2a(d['storage_index'])
u = IFileURI(self.uri)
if isinstance(u, uri.CHKFileURI):
return "CHK-REFRESH:%s" % idlib.b2a(u.storage_index)
return None
def download(self, target):

42
src/allmydata/download.py
View File

@ -9,7 +9,7 @@ from allmydata.util import idlib, mathutil, hashutil
from allmydata.util.assertutil import _assert
from allmydata import codec, hashtree, storage, uri
from allmydata.Crypto.Cipher import AES
from allmydata.interfaces import IDownloadTarget, IDownloader
from allmydata.interfaces import IDownloadTarget, IDownloader, IFileURI
from allmydata.encode import NotEnoughPeersError
class HaveAllPeersError(Exception):
@ -288,14 +288,14 @@ class FileDownloader:
def __init__(self, client, u, downloadable):
self._client = client
d = uri.unpack_uri(u)
self._storage_index = d['storage_index']
self._uri_extension_hash = d['uri_extension_hash']
self._total_shares = d['total_shares']
self._size = d['size']
self._num_needed_shares = d['needed_shares']
u = IFileURI(u)
self._storage_index = u.storage_index
self._uri_extension_hash = u.uri_extension_hash
self._total_shares = u.total_shares
self._size = u.size
self._num_needed_shares = u.needed_shares
self._output = Output(downloadable, d['key'], self._size)
self._output = Output(downloadable, u.key, self._size)
self.active_buckets = {} # k: shnum, v: bucket
self._share_buckets = [] # list of (sharenum, bucket) tuples
@ -609,12 +609,13 @@ class FileDownloader:
return self._output.finish()
class LiteralDownloader:
def __init__(self, client, uri, downloadable):
self._uri = uri
def __init__(self, client, u, downloadable):
self._uri = IFileURI(u)
assert isinstance(self._uri, uri.LiteralFileURI)
self._downloadable = downloadable
def start(self):
data = uri.unpack_lit(self._uri)
data = self._uri.data
self._downloadable.open(len(data))
self._downloadable.write(data)
self._downloadable.close()
@ -625,16 +626,19 @@ class FileName:
implements(IDownloadTarget)
def __init__(self, filename):
self._filename = filename
self.f = None
def open(self, size):
self.f = open(self._filename, "wb")
return self.f
def write(self, data):
self.f.write(data)
def close(self):
self.f.close()
if self.f:
self.f.close()
def fail(self, why):
self.f.close()
os.unlink(self._filename)
if self.f:
self.f.close()
os.unlink(self._filename)
def register_canceller(self, cb):
pass # we won't use it
def finish(self):
@ -690,14 +694,16 @@ class Downloader(service.MultiService):
def download(self, u, t):
assert self.parent
assert self.running
u = IFileURI(u)
t = IDownloadTarget(t)
assert t.write
assert t.close
utype = uri.get_uri_type(u)
if utype == "CHK":
dl = FileDownloader(self.parent, u, t)
elif utype == "LIT":
if isinstance(u, uri.LiteralFileURI):
dl = LiteralDownloader(self.parent, u, t)
elif isinstance(u, uri.CHKFileURI):
dl = FileDownloader(self.parent, u, t)
else:
raise RuntimeError("I don't know how to download a %s" % u)
d = dl.start()
return d

36
src/allmydata/interfaces.py
View File

@ -228,6 +228,38 @@ class RIVirtualDriveServer(RemoteInterface):
"""
class IURI(Interface):
def init_from_string(uri):
"""Accept a string (as created by my to_string() method) and populate
this instance with its data. I am not normally called directly,
please use the module-level uri.from_string() function to convert
arbitrary URI strings into IURI-providing instances."""
def is_readonly():
"""Return False if this URI be used to modify the data. Return True
if this URI cannot be used to modify the data."""
def is_mutable():
"""Return True if the data can be modified by *somebody* (perhaps
someone who has a more powerful URI than this one)."""
def get_readonly():
"""Return another IURI instance, which represents a read-only form of
this one. If is_readonly() is True, this returns self."""
def to_string():
"""Return a string of printable ASCII characters, suitable for
passing into init_from_string."""
class IDirnodeURI(Interface):
"""I am a URI which represents a dirnode."""
class IFileURI(Interface):
"""I am a URI which represents a filenode."""
def get_size():
"""Return the length (in bytes) of the file that I represent."""
class IFileNode(Interface):
def download(target):
"""Download the file's contents to a given IDownloadTarget"""
@ -239,6 +271,8 @@ class IFileNode(Interface):
"""Return the URI that can be used by others to get access to this
file.
"""
def get_size():
"""Return the length (in bytes) of the data this node represents."""
def get_refresh_capability():
"""Return a string that represents the 'refresh capability' for this
@ -788,7 +822,7 @@ class IVirtualDrive(Interface):
"""
def get_node(uri):
"""Transform a URI into an IDirectoryNode or IFileNode.
"""Transform a URI (or IURI) into an IDirectoryNode or IFileNode.
This returns a Deferred that will fire with an instance that provides
either IDirectoryNode or IFileNode, as appropriate."""

20
src/allmydata/scripts/debug.py
View File

@ -91,8 +91,8 @@ def dump_root_dirnode(config, out=sys.stdout, err=sys.stderr):
try:
f = open(root_dirnode_file, "rb")
key = f.read()
rooturi = uri.pack_dirnode_uri("fakeFURL", key)
print >>out, rooturi
rooturi = uri.DirnodeURI("fakeFURL", key)
print >>out, rooturi.to_string()
return 0
except EnvironmentError:
print >>out, "unable to read root dirnode file from %s" % \
@ -100,22 +100,24 @@ def dump_root_dirnode(config, out=sys.stdout, err=sys.stderr):
return 1
def dump_directory_node(config, out=sys.stdout, err=sys.stderr):
from allmydata import uri, dirnode
from allmydata import dirnode
from allmydata.util import hashutil, idlib
from allmydata.interfaces import IDirnodeURI
basedir = config['basedirs'][0]
dir_uri = config['uri']
dir_uri = IDirnodeURI(config['uri'])
verbose = config['verbose']
furl, key = uri.unpack_dirnode_uri(dir_uri)
if uri.is_mutable_dirnode_uri(dir_uri):
wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key)
if dir_uri.is_readonly():
wk, we, rk, index = \
hashutil.generate_dirnode_keys_from_readkey(dir_uri.readkey)
else:
wk, we, rk, index = hashutil.generate_dirnode_keys_from_readkey(key)
wk, we, rk, index = \
hashutil.generate_dirnode_keys_from_writekey(dir_uri.writekey)
filename = os.path.join(basedir, "vdrive", idlib.b2a(index))
print >>out
print >>out, "dirnode uri: %s" % dir_uri
print >>out, "dirnode uri: %s" % dir_uri.to_string()
print >>out, "filename : %s" % filename
print >>out, "index : %s" % idlib.b2a(index)
if wk:

34
src/allmydata/test/test_dirnode.py
View File

@ -6,7 +6,7 @@ from twisted.internet import defer
from twisted.python import failure
from allmydata import uri, dirnode
from allmydata.util import hashutil
from allmydata.interfaces import IDirectoryNode
from allmydata.interfaces import IDirectoryNode, IDirnodeURI
from allmydata.scripts import runner
from allmydata.dirnode import VirtualDriveServer, \
ChildAlreadyPresentError, BadWriteEnablerError, NoPublicRootError
@ -20,13 +20,13 @@ class DirectoryNode(unittest.TestCase):
vds.set_furl("myFURL")
root_uri = vds.get_public_root_uri()
self.failUnless(uri.is_dirnode_uri(root_uri))
self.failUnless(uri.is_mutable_dirnode_uri(root_uri))
furl, key = uri.unpack_dirnode_uri(root_uri)
self.failUnlessEqual(furl, "myFURL")
self.failUnlessEqual(len(key), hashutil.KEYLEN)
u = IDirnodeURI(root_uri)
self.failIf(u.is_readonly())
self.failUnlessEqual(u.furl, "myFURL")
self.failUnlessEqual(len(u.writekey), hashutil.KEYLEN)
wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key)
wk, we, rk, index = \
hashutil.generate_dirnode_keys_from_writekey(u.writekey)
empty_list = vds.list(index)
self.failUnlessEqual(empty_list, [])
@ -78,10 +78,10 @@ class DirectoryNode(unittest.TestCase):
vds2 = VirtualDriveServer(basedir)
vds2.set_furl("myFURL")
root_uri2 = vds.get_public_root_uri()
self.failUnless(uri.is_mutable_dirnode_uri(root_uri2))
furl2, key2 = uri.unpack_dirnode_uri(root_uri2)
u2 = IDirnodeURI(root_uri2)
self.failIf(u2.is_readonly())
(wk2, we2, rk2, index2) = \
hashutil.generate_dirnode_keys_from_writekey(key2)
hashutil.generate_dirnode_keys_from_writekey(u2.writekey)
self.failUnlessEqual(sorted(vds2.list(index2)),
[ ("name2", "", "read2"),
])
@ -173,8 +173,18 @@ class Test(unittest.TestCase):
self.failUnlessEqual(res, {})
d.addCallback(_listed)
file1 = uri.pack_uri("11" + " "*30, "k"*16, "e"*32, 25, 100, 12345)
file2 = uri.pack_uri("2i" + " "*30, "k"*16, "e"*32, 25, 100, 12345)
file1 = uri.CHKFileURI(storage_index="11" + " "*30,
key="k"*16,
uri_extension_hash="e"*32,
needed_shares=25,
total_shares=100,
size=12345).to_string()
file2 = uri.CHKFileURI(storage_index="2i" + " "*30,
key="k"*16,
uri_extension_hash="e"*32,
needed_shares=25,
total_shares=100,
size=12345).to_string()
file2_node = dirnode.FileNode(file2, None)
d.addCallback(lambda res: rootnode.set_uri("foo", file1))
# root/

17
src/allmydata/test/test_encode.py
View File

@ -3,12 +3,9 @@ from zope.interface import implements
from twisted.trial import unittest
from twisted.internet import defer
from twisted.python.failure import Failure
from allmydata import encode, upload, download, hashtree
from allmydata import encode, upload, download, hashtree, uri
from allmydata.util import hashutil
from allmydata.uri import pack_uri
from allmydata.Crypto.Cipher import AES
from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader
from cStringIO import StringIO
class LostPeerError(Exception):
pass
@ -308,12 +305,12 @@ class Roundtrip(unittest.TestCase):
if "corrupt_key" in recover_mode:
key = flip_bit(key)
URI = pack_uri(storage_index="S" * 32,
key=key,
uri_extension_hash=uri_extension_hash,
needed_shares=e.required_shares,
total_shares=e.num_shares,
size=e.file_size)
URI = uri.CHKFileURI(storage_index="S" * 32,
key=key,
uri_extension_hash=uri_extension_hash,
needed_shares=e.required_shares,
total_shares=e.num_shares,
size=e.file_size).to_string()
client = None
target = download.Data()
fd = download.FileDownloader(client, URI, target)

14
src/allmydata/test/test_system.py
View File

@ -8,7 +8,7 @@ from allmydata import client, uri, download, upload
from allmydata.introducer_and_vdrive import IntroducerAndVdrive
from allmydata.util import idlib, fileutil, testutil
from allmydata.scripts import runner
from allmydata.interfaces import IDirectoryNode, IFileNode
from allmydata.interfaces import IDirectoryNode, IFileNode, IFileURI
from allmydata.dirnode import NotMutableError
from foolscap.eventual import flushEventualQueue
from twisted.python import log
@ -224,10 +224,14 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase):
def mangle_uri(self, gooduri):
# change the storage index, which means we'll be asking about the
# wrong file, so nobody will have any shares
d = uri.unpack_uri(gooduri)
assert len(d['storage_index']) == 32
d['storage_index'] = self.flip_bit(d['storage_index'])
return uri.pack_uri(**d)
u = IFileURI(gooduri)
u2 = uri.CHKFileURI(storage_index=self.flip_bit(u.storage_index),
key=u.key,
uri_extension_hash=u.uri_extension_hash,
needed_shares=u.needed_shares,
total_shares=u.total_shares,
size=u.size)
return u2.to_string()
# TODO: add a test which mangles the uri_extension_hash instead, and
# should fail due to not being able to get a valid uri_extension block.

30
src/allmydata/test/test_upload.py
View File

@ -5,8 +5,8 @@ from twisted.python.failure import Failure
from twisted.internet import defer
from cStringIO import StringIO
from allmydata import upload, encode
from allmydata.uri import unpack_uri, unpack_lit
from allmydata import upload, encode, uri
from allmydata.interfaces import IFileURI
from allmydata.util.assertutil import precondition
from foolscap import eventual
@ -154,21 +154,19 @@ class GoodServer(unittest.TestCase):
self.u.running = True
self.u.parent = self.node
def _check_small(self, uri, size):
self.failUnless(isinstance(uri, str))
self.failUnless(uri.startswith("URI:LIT:"))
d = unpack_lit(uri)
self.failUnlessEqual(len(d), size)
def _check_small(self, newuri, size):
u = IFileURI(newuri)
self.failUnless(isinstance(u, uri.LiteralFileURI))
self.failUnlessEqual(len(u.data), size)
def _check_large(self, uri, size):
self.failUnless(isinstance(uri, str))
self.failUnless(uri.startswith("URI:"))
d = unpack_uri(uri)
self.failUnless(isinstance(d['storage_index'], str))
self.failUnlessEqual(len(d['storage_index']), 32)
self.failUnless(isinstance(d['key'], str))
self.failUnlessEqual(len(d['key']), 16)
self.failUnlessEqual(d['size'], size)
def _check_large(self, newuri, size):
u = IFileURI(newuri)
self.failUnless(isinstance(u, uri.CHKFileURI))
self.failUnless(isinstance(u.storage_index, str))
self.failUnlessEqual(len(u.storage_index), 32)
self.failUnless(isinstance(u.key, str))
self.failUnlessEqual(len(u.key), 16)
self.failUnlessEqual(u.size, size)
def get_data(self, size):
return DATA[:size]

150
src/allmydata/test/test_uri.py
View File

@ -2,23 +2,50 @@
from twisted.trial import unittest
from allmydata import uri
from allmydata.util import hashutil
from allmydata.interfaces import IURI, IFileURI, IDirnodeURI
class LIT(unittest.TestCase):
class Literal(unittest.TestCase):
def test_pack(self):
data = "This is some small data"
u = uri.pack_lit(data)
self.failUnlessEqual(uri.get_uri_type(u), "LIT")
self.failUnlessEqual(uri.unpack_lit(u), data)
self.failUnless(uri.is_filenode_uri(u))
self.failUnlessEqual(uri.get_filenode_size(u), len(data))
u = uri.LiteralFileURI(data)
self.failUnless(IURI.providedBy(u))
self.failUnless(IFileURI.providedBy(u))
self.failIf(IDirnodeURI.providedBy(u))
self.failUnlessEqual(u.data, data)
self.failUnlessEqual(u.get_size(), len(data))
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
u2 = uri.from_string(u.to_string())
self.failUnless(IURI.providedBy(u2))
self.failUnless(IFileURI.providedBy(u2))
self.failIf(IDirnodeURI.providedBy(u2))
self.failUnlessEqual(u2.data, data)
self.failUnlessEqual(u2.get_size(), len(data))
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
def test_nonascii(self):
data = "This contains \x00 and URI:LIT: and \n, oh my."
u = uri.pack_lit(data)
self.failUnlessEqual(uri.get_uri_type(u), "LIT")
self.failUnlessEqual(uri.unpack_lit(u), data)
u = uri.LiteralFileURI(data)
self.failUnless(IURI.providedBy(u))
self.failUnless(IFileURI.providedBy(u))
self.failIf(IDirnodeURI.providedBy(u))
self.failUnlessEqual(u.data, data)
self.failUnlessEqual(u.get_size(), len(data))
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
class CHK(unittest.TestCase):
u2 = uri.from_string(u.to_string())
self.failUnless(IURI.providedBy(u2))
self.failUnless(IFileURI.providedBy(u2))
self.failIf(IDirnodeURI.providedBy(u2))
self.failUnlessEqual(u2.data, data)
self.failUnlessEqual(u2.get_size(), len(data))
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
class CHKFile(unittest.TestCase):
def test_pack(self):
storage_index = hashutil.tagged_hash("foo", "bar")
key = "\x00" * 16
@ -26,23 +53,42 @@ class CHK(unittest.TestCase):
needed_shares = 25
total_shares = 100
size = 1234
u = uri.pack_uri(storage_index=storage_index,
key=key,
uri_extension_hash=uri_extension_hash,
needed_shares=needed_shares,
total_shares=total_shares,
size=size)
self.failUnlessEqual(uri.get_uri_type(u), "CHK")
d = uri.unpack_uri(u)
self.failUnlessEqual(d['storage_index'], storage_index)
self.failUnlessEqual(d['key'], key)
self.failUnlessEqual(d['uri_extension_hash'], uri_extension_hash)
self.failUnlessEqual(d['needed_shares'], needed_shares)
self.failUnlessEqual(d['total_shares'], total_shares)
self.failUnlessEqual(d['size'], size)
u = uri.CHKFileURI(storage_index=storage_index,
key=key,
uri_extension_hash=uri_extension_hash,
needed_shares=needed_shares,
total_shares=total_shares,
size=size)
self.failUnlessEqual(u.storage_index, storage_index)
self.failUnlessEqual(u.key, key)
self.failUnlessEqual(u.uri_extension_hash, uri_extension_hash)
self.failUnlessEqual(u.needed_shares, needed_shares)
self.failUnlessEqual(u.total_shares, total_shares)
self.failUnlessEqual(u.size, size)
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
self.failUnless(IURI.providedBy(u))
self.failUnless(IFileURI.providedBy(u))
self.failIf(IDirnodeURI.providedBy(u))
self.failUnlessEqual(u.get_size(), 1234)
self.failUnless(u.is_readonly())
self.failIf(u.is_mutable())
self.failUnless(uri.is_filenode_uri(u))
self.failUnlessEqual(uri.get_filenode_size(u), size)
u2 = uri.from_string(u.to_string())
self.failUnlessEqual(u2.storage_index, storage_index)
self.failUnlessEqual(u2.key, key)
self.failUnlessEqual(u2.uri_extension_hash, uri_extension_hash)
self.failUnlessEqual(u2.needed_shares, needed_shares)
self.failUnlessEqual(u2.total_shares, total_shares)
self.failUnlessEqual(u2.size, size)
self.failUnless(u2.is_readonly())
self.failIf(u2.is_mutable())
self.failUnless(IURI.providedBy(u2))
self.failUnless(IFileURI.providedBy(u2))
self.failIf(IDirnodeURI.providedBy(u2))
self.failUnlessEqual(u2.get_size(), 1234)
self.failUnless(u2.is_readonly())
self.failIf(u2.is_mutable())
class Extension(unittest.TestCase):
def test_pack(self):
@ -64,26 +110,40 @@ class Dirnode(unittest.TestCase):
furl = "pb://stuff@morestuff:stuff/andstuff"
writekey = "\x01" * 16
u = uri.pack_dirnode_uri(furl, writekey)
self.failUnless(uri.is_dirnode_uri(u))
self.failIf(uri.is_dirnode_uri("NOT A DIRNODE URI"))
self.failIf(uri.is_dirnode_uri("URI:stuff"))
self.failUnless(uri.is_mutable_dirnode_uri(u))
self.failIf(uri.is_mutable_dirnode_uri("NOT A DIRNODE URI"))
self.failIf(uri.is_mutable_dirnode_uri("URI:stuff"))
self.failUnlessEqual(uri.get_uri_type(u), "DIR")
u = uri.DirnodeURI(furl, writekey)
self.failUnlessEqual(u.furl, furl)
self.failUnlessEqual(u.writekey, writekey)
self.failIf(u.is_readonly())
self.failUnless(u.is_mutable())
self.failUnless(IURI.providedBy(u))
self.failIf(IFileURI.providedBy(u))
self.failUnless(IDirnodeURI.providedBy(u))
rou = uri.make_immutable_dirnode_uri(u)
self.failUnless(uri.is_dirnode_uri(rou))
self.failIf(uri.is_mutable_dirnode_uri(rou))
self.failUnlessEqual(uri.get_uri_type(rou), "DIR-RO")
u2 = uri.from_string(u.to_string())
self.failUnlessEqual(u2.furl, furl)
self.failUnlessEqual(u2.writekey, writekey)
self.failIf(u2.is_readonly())
self.failUnless(u2.is_mutable())
self.failUnless(IURI.providedBy(u2))
self.failIf(IFileURI.providedBy(u2))
self.failUnless(IDirnodeURI.providedBy(u2))
d = uri.unpack_dirnode_uri(u)
self.failUnlessEqual(d[0], furl)
self.failUnlessEqual(d[1], writekey)
u3 = u2.get_readonly()
readkey = hashutil.dir_read_key_hash(writekey)
self.failUnlessEqual(u3.furl, furl)
self.failUnlessEqual(u3.readkey, readkey)
self.failUnless(u3.is_readonly())
self.failUnless(u3.is_mutable())
self.failUnless(IURI.providedBy(u3))
self.failIf(IFileURI.providedBy(u3))
self.failUnless(IDirnodeURI.providedBy(u3))
d2 = uri.unpack_dirnode_uri(rou)
self.failUnlessEqual(d2[0], furl)
rk = hashutil.dir_read_key_hash(writekey)
self.failUnlessEqual(d2[1], rk)
u4 = uri.ReadOnlyDirnodeURI(furl, readkey)
self.failUnlessEqual(u4.furl, furl)
self.failUnlessEqual(u4.readkey, readkey)
self.failUnless(u4.is_readonly())
self.failUnless(u4.is_mutable())
self.failUnless(IURI.providedBy(u4))
self.failIf(IFileURI.providedBy(u4))
self.failUnless(IDirnodeURI.providedBy(u4))

52
src/allmydata/test/test_web.py
View File

@ -48,6 +48,17 @@ class MyDownloader(service.Service):
uri_counter = itertools.count()
def make_newuri(data):
n = uri_counter.next()
assert len(str(n)) < 5
newuri = uri.CHKFileURI(storage_index="SI%05d" % n + "i"*25,
key="K"*16,
uri_extension_hash="EH" + "h"*30,
needed_shares=25,
total_shares=100,
size=len(data))
return newuri.to_string()
class MyUploader(service.Service):
implements(interfaces.IUploader)
name = "uploader"
@ -59,31 +70,26 @@ class MyUploader(service.Service):
d.addCallback(lambda size: uploadable.read(size))
d.addCallback(lambda data: "".join(data))
def _got_data(data):
uri = str(uri_counter.next())
self.files[uri] = data
newuri = make_newuri(data)
self.files[newuri] = data
uploadable.close()
d.addCallback(_got_data)
return d
class MyDirectoryNode(dirnode.MutableDirectoryNode):
def __init__(self, nodes, files, client, uri=None):
def __init__(self, nodes, files, client, myuri=None):
self._my_nodes = nodes
self._my_files = files
self._my_client = client
if uri is None:
uri = "URI:DIR:stuff/%s" % str(uri_counter.next())
self._uri = str(uri)
if myuri is None:
u = uri.DirnodeURI("furl", "idx%s" % str(uri_counter.next()))
myuri = u.to_string()
self._uri = myuri
self._my_nodes[self._uri] = self
self.children = {}
self._mutable = True
def get_immutable_uri(self):
return self.get_uri() + "RO"
def get_refresh_capability(self):
return "refresh:" + self.get_uri()
def get(self, name):
def _try():
uri = self.children[name]
@ -101,12 +107,12 @@ class MyDirectoryNode(dirnode.MutableDirectoryNode):
d.addCallback(lambda size: uploadable.read(size))
d.addCallback(lambda data: "".join(data))
def _got_data(data):
uri = str(uri_counter.next())
self._my_files[uri] = data
self._my_nodes[uri] = MyFileNode(uri, self._my_client)
self.children[name] = uri
newuri = make_newuri(data)
self._my_files[newuri] = data
self._my_nodes[newuri] = MyFileNode(newuri, self._my_client)
self.children[name] = newuri
uploadable.close()
return self._my_nodes[uri]
return self._my_nodes[newuri]
d.addCallback(_got_data)
return d
@ -214,10 +220,12 @@ class Web(unittest.TestCase):
def makefile(self, number):
n = str(number)
assert len(n) == 1
newuri = uri.pack_uri("SI" + n*30,
"K" + n*15,
"EH" + n*30,
25, 100, 123+number)
newuri = uri.CHKFileURI(storage_index="SI" + n*30,
key="K" + n*15,
uri_extension_hash="EH" + n*30,
needed_shares=25,
total_shares=100,
size=123+number).to_string()
assert newuri not in self.nodes
assert newuri not in self.files
node = MyFileNode(newuri, self.s)
@ -230,7 +238,7 @@ class Web(unittest.TestCase):
n = str(number)
assert len(n) == 1
contents = "small data %s\n" % n
newuri = uri.pack_lit(contents)
newuri = uri.LiteralFileURI(contents).to_string()
assert newuri not in self.nodes
assert newuri not in self.files
node = MyFileNode(newuri, self.s)

21
src/allmydata/upload.py
View File

@ -7,8 +7,7 @@ from twisted.application import service
from foolscap import Referenceable
from allmydata.util import idlib, hashutil
from allmydata import encode, storage, hashtree
from allmydata.uri import pack_uri, pack_lit
from allmydata import encode, storage, hashtree, uri
from allmydata.interfaces import IUploadable, IUploader
from cStringIO import StringIO
@ -321,13 +320,14 @@ class CHKUploader:
self._encoder.set_shareholders(buckets)
def _compute_uri(self, uri_extension_hash):
return pack_uri(storage_index=self._storage_index,
key=self._encryption_key,
uri_extension_hash=uri_extension_hash,
needed_shares=self.needed_shares,
total_shares=self.total_shares,
size=self._size,
)
u = uri.CHKFileURI(storage_index=self._storage_index,
key=self._encryption_key,
uri_extension_hash=uri_extension_hash,
needed_shares=self.needed_shares,
total_shares=self.total_shares,
size=self._size,
)
return u.to_string()
def read_this_many_bytes(uploadable, size, prepend_data=[]):
if size == 0:
@ -359,7 +359,8 @@ class LiteralUploader:
def start(self):
d = self._uploadable.get_size()
d.addCallback(lambda size: read_this_many_bytes(self._uploadable, size))
d.addCallback(lambda data: pack_lit("".join(data)))
d.addCallback(lambda data: uri.LiteralFileURI("".join(data)))
d.addCallback(lambda u: u.to_string())
return d
def close(self):

272
src/allmydata/uri.py
View File

@ -1,66 +1,202 @@
import re
from zope.interface import implements
from twisted.python.components import registerAdapter
from allmydata.util import idlib, hashutil
def get_uri_type(uri):
assert uri.startswith("URI:")
if uri.startswith("URI:DIR:"):
return "DIR"
if uri.startswith("URI:DIR-RO:"):
return "DIR-RO"
if uri.startswith("URI:LIT:"):
return "LIT"
return "CHK"
def is_filenode_uri(uri):
return get_uri_type(uri) in ("LIT", "CHK")
def get_filenode_size(uri):
assert is_filenode_uri(uri)
t = get_uri_type(uri)
if t == "LIT":
return len(unpack_lit(uri))
return unpack_uri(uri)['size']
from allmydata.interfaces import IURI, IDirnodeURI, IFileURI
# the URI shall be an ascii representation of the file. It shall contain
# enough information to retrieve and validate the contents. It shall be
# expressed in a limited character set (namely [TODO]).
def pack_uri(storage_index, key, uri_extension_hash,
needed_shares, total_shares, size):
# applications should pass keyword parameters into this
assert isinstance(storage_index, str)
assert len(storage_index) == 32 # sha256 hash
assert isinstance(uri_extension_hash, str)
assert len(uri_extension_hash) == 32 # sha56 hash
class _BaseURI:
def __hash__(self):
return hash((self.__class__, self.to_string()))
def __cmp__(self, them):
if cmp(type(self), type(them)):
return cmp(type(self), type(them))
if cmp(self.__class__, them.__class__):
return cmp(self.__class__, them.__class__)
return cmp(self.to_string(), them.to_string())
assert isinstance(key, str)
assert len(key) == 16 # AES-128
assert isinstance(needed_shares, int)
assert isinstance(total_shares, int)
assert isinstance(size, (int,long))
class CHKFileURI(_BaseURI):
implements(IURI, IFileURI)
return "URI:%s:%s:%s:%d:%d:%d" % (idlib.b2a(storage_index), idlib.b2a(key),
idlib.b2a(uri_extension_hash),
needed_shares, total_shares, size)
def __init__(self, **kwargs):
# construct me with kwargs, since there are so many of them
if not kwargs:
return
for name in ("storage_index", "key", "uri_extension_hash",
"needed_shares", "total_shares", "size"):
value = kwargs[name]
setattr(self, name, value)
def init_from_string(self, uri):
assert uri.startswith("URI:CHK:"), uri
d = {}
(header_uri, header_chk,
storage_index_s, key_s, uri_extension_hash_s,
needed_shares_s, total_shares_s, size_s) = uri.split(":")
assert header_uri == "URI"
assert header_chk == "CHK"
self.storage_index = idlib.a2b(storage_index_s)
self.key = idlib.a2b(key_s)
self.uri_extension_hash = idlib.a2b(uri_extension_hash_s)
self.needed_shares = int(needed_shares_s)
self.total_shares = int(total_shares_s)
self.size = int(size_s)
return self
def unpack_uri(uri):
assert uri.startswith("URI:"), uri
d = {}
(header,
storage_index_s, key_s, uri_extension_hash_s,
needed_shares_s, total_shares_s, size_s) = uri.split(":")
assert header == "URI"
d['storage_index'] = idlib.a2b(storage_index_s)
d['key'] = idlib.a2b(key_s)
d['uri_extension_hash'] = idlib.a2b(uri_extension_hash_s)
d['needed_shares'] = int(needed_shares_s)
d['total_shares'] = int(total_shares_s)
d['size'] = int(size_s)
return d
def to_string(self):
assert isinstance(self.storage_index, str)
assert len(self.storage_index) == 32 # sha256 hash
assert isinstance(self.uri_extension_hash, str)
assert len(self.uri_extension_hash) == 32 # sha56 hash
assert isinstance(self.key, str)
assert len(self.key) == 16 # AES-128
assert isinstance(self.needed_shares, int)
assert isinstance(self.total_shares, int)
assert isinstance(self.size, (int,long))
return ("URI:CHK:%s:%s:%s:%d:%d:%d" %
(idlib.b2a(self.storage_index),
idlib.b2a(self.key),
idlib.b2a(self.uri_extension_hash),
self.needed_shares,
self.total_shares,
self.size))
def is_readonly(self):
return True
def is_mutable(self):
return False
def get_readonly(self):
return self
def get_size(self):
return self.size
class LiteralFileURI(_BaseURI):
implements(IURI, IFileURI)
def __init__(self, data=None):
if data is not None:
self.data = data
def init_from_string(self, uri):
assert uri.startswith("URI:LIT:")
data_s = uri[len("URI:LIT:"):]
self.data = idlib.a2b(data_s)
return self
def to_string(self):
return "URI:LIT:%s" % idlib.b2a(self.data)
def is_readonly(self):
return True
def is_mutable(self):
return False
def get_readonly(self):
return self
def get_size(self):
return len(self.data)
class DirnodeURI(_BaseURI):
implements(IURI, IDirnodeURI)
def __init__(self, furl=None, writekey=None):
if furl is not None or writekey is not None:
assert furl is not None
assert writekey is not None
self.furl = furl
self.writekey = writekey
def init_from_string(self, uri):
# URI:DIR:furl:key
# but note that the furl contains colons
prefix = "URI:DIR:"
assert uri.startswith(prefix)
uri = uri[len(prefix):]
colon = uri.rindex(":")
self.furl = uri[:colon]
self.writekey = idlib.a2b(uri[colon+1:])
return self
def to_string(self):
return "URI:DIR:%s:%s" % (self.furl, idlib.b2a(self.writekey))
def is_readonly(self):
return False
def is_mutable(self):
return True
def get_readonly(self):
u = ReadOnlyDirnodeURI()
u.furl = self.furl
u.readkey = hashutil.dir_read_key_hash(self.writekey)
return u
class ReadOnlyDirnodeURI(_BaseURI):
implements(IURI, IDirnodeURI)
def __init__(self, furl=None, readkey=None):
if furl is not None or readkey is not None:
assert furl is not None
assert readkey is not None
self.furl = furl
self.readkey = readkey
def init_from_string(self, uri):
# URI:DIR-RO:furl:key
# but note that the furl contains colons
prefix = "URI:DIR-RO:"
assert uri.startswith(prefix)
uri = uri[len(prefix):]
colon = uri.rindex(":")
self.furl = uri[:colon]
self.readkey = idlib.a2b(uri[colon+1:])
return self
def to_string(self):
return "URI:DIR-RO:%s:%s" % (self.furl, idlib.b2a(self.readkey))
def is_readonly(self):
return True
def is_mutable(self):
return True
def get_readonly(self):
return self
def from_string(s):
if s.startswith("URI:CHK:"):
return CHKFileURI().init_from_string(s)
elif s.startswith("URI:LIT:"):
return LiteralFileURI().init_from_string(s)
elif s.startswith("URI:DIR:"):
return DirnodeURI().init_from_string(s)
elif s.startswith("URI:DIR-RO:"):
return ReadOnlyDirnodeURI().init_from_string(s)
else:
raise RuntimeError("unknown URI type: %s.." % s[:10])
registerAdapter(from_string, str, IURI)
def from_string_dirnode(s):
u = from_string(s)
assert IDirnodeURI.providedBy(u)
return u
registerAdapter(from_string_dirnode, str, IDirnodeURI)
def from_string_filenode(s):
u = from_string(s)
assert IFileURI.providedBy(u)
return u
registerAdapter(from_string_filenode, str, IFileURI)
def pack_extension(data):
@ -108,39 +244,3 @@ def unpack_extension_readable(data):
unpacked[k] = idlib.b2a(unpacked[k])
return unpacked
def pack_lit(data):
return "URI:LIT:%s" % idlib.b2a(data)
def unpack_lit(uri):
assert uri.startswith("URI:LIT:")
data_s = uri[len("URI:LIT:"):]
return idlib.a2b(data_s)
def is_dirnode_uri(uri):
return uri.startswith("URI:DIR:") or uri.startswith("URI:DIR-RO:")
def is_mutable_dirnode_uri(uri):
return uri.startswith("URI:DIR:")
def unpack_dirnode_uri(uri):
assert is_dirnode_uri(uri)
# URI:DIR:furl:key
# but note that the furl contains colons
for prefix in ("URI:DIR:", "URI:DIR-RO:"):
if uri.startswith(prefix):
uri = uri[len(prefix):]
break
else:
assert 0
colon = uri.rindex(":")
furl = uri[:colon]
key = uri[colon+1:]
return furl, idlib.a2b(key)
def make_immutable_dirnode_uri(mutable_uri):
assert is_mutable_dirnode_uri(mutable_uri)
furl, writekey = unpack_dirnode_uri(mutable_uri)
readkey = hashutil.dir_read_key_hash(writekey)
return "URI:DIR-RO:%s:%s" % (furl, idlib.b2a(readkey))
def pack_dirnode_uri(furl, writekey):
return "URI:DIR:%s:%s" % (furl, idlib.b2a(writekey))

7
src/allmydata/vdrive.py
View File

@ -2,8 +2,8 @@
import os
from twisted.application import service
from zope.interface import implements
from allmydata.interfaces import IVirtualDrive
from allmydata import dirnode, uri
from allmydata.interfaces import IVirtualDrive, IDirnodeURI, IURI
from allmydata import dirnode
from twisted.internet import defer
class NoGlobalVirtualDriveError(Exception):
@ -88,7 +88,8 @@ class VirtualDrive(service.MultiService):
return self.get_node(self._private_uri)
def get_node(self, node_uri):
if uri.is_dirnode_uri(node_uri):
node_uri = IURI(node_uri)
if IDirnodeURI.providedBy(node_uri):
return dirnode.create_directory_node(self.parent, node_uri)
else:
return defer.succeed(dirnode.FileNode(node_uri, self.parent))

23
src/allmydata/webish.py
View File

@ -8,9 +8,8 @@ from nevow import inevow, rend, loaders, appserver, url, tags as T
from nevow.static import File as nevow_File # TODO: merge with static.File?
from allmydata.util import idlib, fileutil
import simplejson
from allmydata.uri import unpack_uri, is_dirnode_uri
from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode
from allmydata import upload, download, uri
from allmydata import upload, download
from zope.interface import implements, Interface
import urllib
from formless import webform
@ -103,19 +102,7 @@ class Directory(rend.Page):
T.a(href=dlurl)[html.escape(name)])
ctx.fillSlots("type", "FILE")
#uri = target.uri
#dl_uri_url = url.root.child("download_uri").child(uri)
## add a filename= query argument to give it a Content-Type
#dl_uri_url = dl_uri_url.add("filename", name)
#ctx.fillSlots("uri", T.a(href=dl_uri_url)[html.escape(uri)])
#extract and display file size
try:
size = uri.get_filenode_size(target.get_uri())
except AssertionError:
size = "?"
ctx.fillSlots("size", size)
ctx.fillSlots("size", target.get_size())
text_plain_link = "/uri/%s?filename=foo.txt" % uri_link
text_plain_tag = T.a(href=text_plain_link)["text/plain"]
@ -322,7 +309,7 @@ class FileJSONMetadata(rend.Page):
data = ("filenode",
{'mutable': False,
'uri': file_uri,
'size': uri.get_filenode_size(file_uri),
'size': filenode.get_size(),
})
return simplejson.dumps(data, indent=1)
@ -412,7 +399,7 @@ class DirectoryJSONMetadata(rend.Page):
kiddata = ("filenode",
{'mutable': False,
'uri': kiduri,
'size': uri.get_filenode_size(kiduri),
'size': childnode.get_size(),
})
else:
assert IDirectoryNode.providedBy(childnode)
@ -519,8 +506,6 @@ class POSTHandler(rend.Page):
newuri = req.args["uri"][0]
else:
newuri = req.fields["uri"].value
# sanity checking
assert(is_dirnode_uri(newuri) or unpack_uri(newuri))
d = self._node.set_uri(name, newuri)
def _done(res):
return newuri