merge vdrive.py and filetable.py into a single dirnode.py

This commit is contained in:
Brian Warner 2007-06-26 17:16:58 -07:00
parent 78c2376fa3
commit b11fa20191
8 changed files with 239 additions and 238 deletions

View File

@ -3,7 +3,7 @@ import os, sha, stat, time
from foolscap import Referenceable, SturdyRef
from zope.interface import implements
from allmydata.interfaces import RIClient, IDirectoryNode
from allmydata import node, vdrive, uri
from allmydata import node, uri
from twisted.internet import defer, reactor
from twisted.application.internet import TimerService
@ -16,6 +16,7 @@ from allmydata.download import Downloader
from allmydata.webish import WebishServer
from allmydata.control import ControlServer
from allmydata.introducer import IntroducerClient
from allmydata.dirnode import create_directory_node, create_directory
class Client(node.Node, Referenceable):
implements(RIClient)
@ -123,7 +124,7 @@ class Client(node.Node, Referenceable):
def _got_vdrive_uri(self, root_uri):
furl, wk = uri.unpack_dirnode_uri(root_uri)
self._vdrive_furl = furl
return vdrive.create_directory_node(self, root_uri)
return create_directory_node(self, root_uri)
def _got_vdrive_rootnode(self, rootnode):
self.log("got vdrive root")
@ -145,10 +146,10 @@ class Client(node.Node, Referenceable):
f = open(MY_VDRIVE_URI_FILE, "r")
my_vdrive_uri = f.read().strip()
f.close()
return vdrive.create_directory_node(self, my_vdrive_uri)
return create_directory_node(self, my_vdrive_uri)
except EnvironmentError:
assert self._vdrive_furl
d = vdrive.create_directory(self, self._vdrive_furl)
d = create_directory(self, self._vdrive_furl)
def _got_directory(dirnode):
f = open(MY_VDRIVE_URI_FILE, "w")
f.write(dirnode.get_uri() + "\n")

View File

@ -1,13 +1,120 @@
"""This is the client-side facility to manipulate virtual drives."""
import os.path
from zope.interface import implements
from twisted.application import service
from twisted.internet import defer
from foolscap import Referenceable
from allmydata import uri
from allmydata.interfaces import RIVirtualDriveServer, IDirectoryNode, IFileNode
from allmydata.util import bencode, idlib, hashutil, fileutil
from allmydata.Crypto.Cipher import AES
from allmydata.util import hashutil, idlib
from allmydata.interfaces import IDirectoryNode, IFileNode
# VirtualDriveServer is the side that hosts directory nodes
class BadWriteEnablerError(Exception):
pass
class ChildAlreadyPresentError(Exception):
pass
class NoPublicRootError(Exception):
pass
class VirtualDriveServer(service.MultiService, Referenceable):
implements(RIVirtualDriveServer)
name = "filetable"
def __init__(self, basedir, offer_public_root=True):
service.MultiService.__init__(self)
self._basedir = os.path.abspath(basedir)
fileutil.make_dirs(self._basedir)
self._root = None
if offer_public_root:
rootfile = os.path.join(self._basedir, "root")
if not os.path.exists(rootfile):
write_key = hashutil.random_key()
(wk, we, rk, index) = \
hashutil.generate_dirnode_keys_from_writekey(write_key)
self.create_directory(index, we)
f = open(rootfile, "wb")
f.write(wk)
f.close()
self._root = wk
else:
f = open(rootfile, "rb")
self._root = f.read()
def set_furl(self, myfurl):
self._myfurl = myfurl
def get_public_root_uri(self):
if self._root:
return uri.pack_dirnode_uri(self._myfurl, self._root)
raise NoPublicRootError
remote_get_public_root_uri = get_public_root_uri
def create_directory(self, index, write_enabler):
data = [write_enabler, []]
self._write_to_file(index, data)
return index
remote_create_directory = create_directory
# the file on disk consists of the write_enabler token and a list of
# (H(name), E(name), E(write), E(read)) tuples.
def _read_from_file(self, index):
name = idlib.b2a(index)
data = open(os.path.join(self._basedir, name), "rb").read()
return bencode.bdecode(data)
def _write_to_file(self, index, data):
name = idlib.b2a(index)
f = open(os.path.join(self._basedir, name), "wb")
f.write(bencode.bencode(data))
f.close()
def get(self, index, key):
data = self._read_from_file(index)
for (H_key, E_key, E_write, E_read) in data[1]:
if H_key == key:
return (E_write, E_read)
raise IndexError("unable to find key %s" % idlib.b2a(key))
remote_get = get
def list(self, index):
data = self._read_from_file(index)
response = [ (E_key, E_write, E_read)
for (H_key, E_key, E_write, E_read) in data[1] ]
return response
remote_list = list
def delete(self, index, write_enabler, key):
data = self._read_from_file(index)
if data[0] != write_enabler:
raise BadWriteEnablerError
for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]):
if H_key == key:
del data[1][i]
self._write_to_file(index, data)
return
raise IndexError("unable to find key %s" % idlib.b2a(key))
remote_delete = delete
def set(self, index, write_enabler, key, name, write, read):
data = self._read_from_file(index)
if data[0] != write_enabler:
raise BadWriteEnablerError
# first, see if the key is already present
for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]):
if H_key == key:
raise ChildAlreadyPresentError
# now just append the data
data[1].append( (key, name, write, read) )
self._write_to_file(index, data)
remote_set = set
# whereas ImmutableDirectoryNodes and their support mechanisms live on the
# client side
class NotMutableError(Exception):
pass

View File

@ -1,110 +0,0 @@
import os
from zope.interface import implements
from twisted.application import service
from foolscap import Referenceable
from allmydata.interfaces import RIVirtualDriveServer
from allmydata.util import bencode, idlib, hashutil, fileutil
from allmydata import uri
class BadWriteEnablerError(Exception):
pass
class ChildAlreadyPresentError(Exception):
pass
class NoPublicRootError(Exception):
pass
class VirtualDriveServer(service.MultiService, Referenceable):
implements(RIVirtualDriveServer)
name = "filetable"
def __init__(self, basedir, offer_public_root=True):
service.MultiService.__init__(self)
self._basedir = os.path.abspath(basedir)
fileutil.make_dirs(self._basedir)
self._root = None
if offer_public_root:
rootfile = os.path.join(self._basedir, "root")
if not os.path.exists(rootfile):
write_key = hashutil.random_key()
(wk, we, rk, index) = \
hashutil.generate_dirnode_keys_from_writekey(write_key)
self.create_directory(index, we)
f = open(rootfile, "wb")
f.write(wk)
f.close()
self._root = wk
else:
f = open(rootfile, "rb")
self._root = f.read()
def set_furl(self, myfurl):
self._myfurl = myfurl
def get_public_root_uri(self):
if self._root:
return uri.pack_dirnode_uri(self._myfurl, self._root)
raise NoPublicRootError
remote_get_public_root_uri = get_public_root_uri
def create_directory(self, index, write_enabler):
data = [write_enabler, []]
self._write_to_file(index, data)
return index
remote_create_directory = create_directory
# the file on disk consists of the write_enabler token and a list of
# (H(name), E(name), E(write), E(read)) tuples.
def _read_from_file(self, index):
name = idlib.b2a(index)
data = open(os.path.join(self._basedir, name), "rb").read()
return bencode.bdecode(data)
def _write_to_file(self, index, data):
name = idlib.b2a(index)
f = open(os.path.join(self._basedir, name), "wb")
f.write(bencode.bencode(data))
f.close()
def get(self, index, key):
data = self._read_from_file(index)
for (H_key, E_key, E_write, E_read) in data[1]:
if H_key == key:
return (E_write, E_read)
raise IndexError("unable to find key %s" % idlib.b2a(key))
remote_get = get
def list(self, index):
data = self._read_from_file(index)
response = [ (E_key, E_write, E_read)
for (H_key, E_key, E_write, E_read) in data[1] ]
return response
remote_list = list
def delete(self, index, write_enabler, key):
data = self._read_from_file(index)
if data[0] != write_enabler:
raise BadWriteEnablerError
for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]):
if H_key == key:
del data[1][i]
self._write_to_file(index, data)
return
raise IndexError("unable to find key %s" % idlib.b2a(key))
remote_delete = delete
def set(self, index, write_enabler, key, name, write, read):
data = self._read_from_file(index)
if data[0] != write_enabler:
raise BadWriteEnablerError
# first, see if the key is already present
for i,(H_key, E_key, E_write, E_read) in enumerate(data[1]):
if H_key == key:
raise ChildAlreadyPresentError
# now just append the data
data[1].append( (key, name, write, read) )
self._write_to_file(index, data)
remote_set = set

View File

@ -1,7 +1,7 @@
import os.path
from allmydata import node
from allmydata.filetable import VirtualDriveServer
from allmydata.dirnode import VirtualDriveServer
from allmydata.introducer import Introducer

View File

@ -373,7 +373,7 @@ def dump_root_dirnode(basedir, config, out=sys.stdout, err=sys.stderr):
return 1
def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr):
from allmydata import filetable, vdrive, uri
from allmydata import uri, dirnode
from allmydata.util import hashutil, idlib
dir_uri = config['uri']
verbose = config['verbose']
@ -400,7 +400,7 @@ def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr):
print >>out
vds = filetable.VirtualDriveServer(os.path.join(basedir, "vdrive"), False)
vds = dirnode.VirtualDriveServer(os.path.join(basedir, "vdrive"), False)
data = vds._read_from_file(index)
if we:
if we != data[0]:
@ -412,16 +412,16 @@ def dump_directory_node(basedir, config, out=sys.stdout, err=sys.stderr):
print >>out, " E_key %s" % idlib.b2a(E_key)
print >>out, " E_write %s" % idlib.b2a(E_write)
print >>out, " E_read %s" % idlib.b2a(E_read)
key = vdrive.decrypt(rk, E_key)
key = dirnode.decrypt(rk, E_key)
print >>out, " key %s" % key
if hashutil.dir_name_hash(rk, key) != H_key:
print >>out, " ERROR: H_key does not match"
if wk and E_write:
if len(E_write) < 14:
print >>out, " ERROR: write data is short:", idlib.b2a(E_write)
write = vdrive.decrypt(wk, E_write)
write = dirnode.decrypt(wk, E_write)
print >>out, " write: %s" % write
read = vdrive.decrypt(rk, E_read)
read = dirnode.decrypt(rk, E_read)
print >>out, " read: %s" % read
print >>out

View File

@ -1,12 +1,101 @@
from cStringIO import StringIO
from twisted.trial import unittest
from cStringIO import StringIO
from foolscap import eventual
from twisted.internet import defer
from twisted.python import failure
from allmydata import vdrive, filetable, uri
from allmydata import uri, dirnode
from allmydata.util import hashutil
from allmydata.interfaces import IDirectoryNode
from allmydata.scripts import runner
from foolscap import eventual
from allmydata.dirnode import VirtualDriveServer, \
ChildAlreadyPresentError, BadWriteEnablerError, NoPublicRootError
# test the host-side code
class DirectoryNode(unittest.TestCase):
def test_vdrive_server(self):
basedir = "dirnode_host/DirectoryNode/test_vdrive_server"
vds = VirtualDriveServer(basedir)
vds.set_furl("myFURL")
root_uri = vds.get_public_root_uri()
self.failUnless(uri.is_dirnode_uri(root_uri))
self.failUnless(uri.is_mutable_dirnode_uri(root_uri))
furl, key = uri.unpack_dirnode_uri(root_uri)
self.failUnlessEqual(furl, "myFURL")
self.failUnlessEqual(len(key), hashutil.KEYLEN)
wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key)
empty_list = vds.list(index)
self.failUnlessEqual(empty_list, [])
vds.set(index, we, "key1", "name1", "write1", "read1")
vds.set(index, we, "key2", "name2", "", "read2")
self.failUnlessRaises(ChildAlreadyPresentError,
vds.set,
index, we, "key2", "name2", "write2", "read2")
self.failUnlessRaises(BadWriteEnablerError,
vds.set,
index, "not the write enabler",
"key2", "name2", "write2", "read2")
self.failUnlessEqual(vds.get(index, "key1"),
("write1", "read1"))
self.failUnlessEqual(vds.get(index, "key2"),
("", "read2"))
self.failUnlessRaises(IndexError,
vds.get, index, "key3")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name1", "write1", "read1"),
("name2", "", "read2"),
])
self.failUnlessRaises(BadWriteEnablerError,
vds.delete,
index, "not the write enabler", "name1")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name1", "write1", "read1"),
("name2", "", "read2"),
])
self.failUnlessRaises(IndexError,
vds.delete,
index, we, "key3")
vds.delete(index, we, "key1")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name2", "", "read2"),
])
self.failUnlessRaises(IndexError,
vds.get, index, "key1")
self.failUnlessEqual(vds.get(index, "key2"),
("", "read2"))
vds2 = VirtualDriveServer(basedir)
vds2.set_furl("myFURL")
root_uri2 = vds.get_public_root_uri()
self.failUnless(uri.is_mutable_dirnode_uri(root_uri2))
furl2, key2 = uri.unpack_dirnode_uri(root_uri2)
(wk2, we2, rk2, index2) = \
hashutil.generate_dirnode_keys_from_writekey(key2)
self.failUnlessEqual(sorted(vds2.list(index2)),
[ ("name2", "", "read2"),
])
def test_no_root(self):
basedir = "dirnode_host/DirectoryNode/test_no_root"
vds = VirtualDriveServer(basedir, offer_public_root=False)
vds.set_furl("myFURL")
self.failUnlessRaises(NoPublicRootError,
vds.get_public_root_uri)
# and the client-side too
class LocalReference:
def __init__(self, target):
@ -34,10 +123,10 @@ class MyClient:
class Test(unittest.TestCase):
def test_create_directory(self):
basedir = "vdrive/test_create_directory/vdrive"
vds = filetable.VirtualDriveServer(basedir)
vds = dirnode.VirtualDriveServer(basedir)
vds.set_furl("myFURL")
self.client = client = MyClient(vds, "myFURL")
d = vdrive.create_directory(client, "myFURL")
d = dirnode.create_directory(client, "myFURL")
def _created(node):
self.failUnless(IDirectoryNode.providedBy(node))
self.failUnless(node.is_mutable())
@ -46,13 +135,13 @@ class Test(unittest.TestCase):
def test_one(self):
self.basedir = basedir = "vdrive/test_one/vdrive"
vds = filetable.VirtualDriveServer(basedir)
vds = dirnode.VirtualDriveServer(basedir)
vds.set_furl("myFURL")
root_uri = vds.get_public_root_uri()
self.client = client = MyClient(vds, "myFURL")
d1 = vdrive.create_directory_node(client, root_uri)
d2 = vdrive.create_directory_node(client, root_uri)
d1 = dirnode.create_directory_node(client, root_uri)
d2 = dirnode.create_directory_node(client, root_uri)
d = defer.gatherResults( [d1,d2] )
d.addCallback(self._test_one_1)
return d
@ -64,7 +153,7 @@ class Test(unittest.TestCase):
self.rootnode = rootnode = rootnode1
self.failUnless(rootnode.is_mutable())
self.readonly_uri = rootnode.get_immutable_uri()
d = vdrive.create_directory_node(self.client, self.readonly_uri)
d = dirnode.create_directory_node(self.client, self.readonly_uri)
d.addCallback(self._test_one_2)
return d
@ -86,7 +175,7 @@ class Test(unittest.TestCase):
file1 = uri.pack_uri("i"*32, "k"*16, "e"*32, 25, 100, 12345)
file2 = uri.pack_uri("i"*31 + "2", "k"*16, "e"*32, 25, 100, 12345)
file2_node = vdrive.FileNode(file2, None)
file2_node = dirnode.FileNode(file2, None)
d.addCallback(lambda res: rootnode.set_uri("foo", file1))
# root/
# root/foo =file1
@ -95,13 +184,13 @@ class Test(unittest.TestCase):
def _listed2(res):
self.failUnlessEqual(res.keys(), ["foo"])
file1_node = res["foo"]
self.failUnless(isinstance(file1_node, vdrive.FileNode))
self.failUnless(isinstance(file1_node, dirnode.FileNode))
self.failUnlessEqual(file1_node.uri, file1)
d.addCallback(_listed2)
d.addCallback(lambda res: rootnode.get("foo"))
def _got_foo(res):
self.failUnless(isinstance(res, vdrive.FileNode))
self.failUnless(isinstance(res, dirnode.FileNode))
self.failUnlessEqual(res.uri, file1)
d.addCallback(_got_foo)
@ -126,7 +215,7 @@ class Test(unittest.TestCase):
# make sure the objects can be used as dict keys
testdict = {res["foo"]: 1, res["bar"]: 2}
bar_node = res["bar"]
self.failUnless(isinstance(bar_node, vdrive.MutableDirectoryNode))
self.failUnless(isinstance(bar_node, dirnode.MutableDirectoryNode))
self.bar_node = bar_node
bar_ro_uri = bar_node.get_immutable_uri()
return rootnode.set_uri("bar-ro", bar_ro_uri)
@ -171,18 +260,18 @@ class Test(unittest.TestCase):
# try to add a file to bar-ro, should get exception
d.addCallback(lambda res:
self.bar_node_readonly.set_uri("file3", file2))
d.addBoth(self.shouldFail, vdrive.NotMutableError,
d.addBoth(self.shouldFail, dirnode.NotMutableError,
"bar-ro.set('file3')")
# try to delete a file from bar-ro, should get exception
d.addCallback(lambda res: self.bar_node_readonly.delete("file2"))
d.addBoth(self.shouldFail, vdrive.NotMutableError,
d.addBoth(self.shouldFail, dirnode.NotMutableError,
"bar-ro.delete('file2')")
# try to mkdir in bar-ro, should get exception
d.addCallback(lambda res:
self.bar_node_readonly.create_empty_directory("boffo"))
d.addBoth(self.shouldFail, vdrive.NotMutableError,
d.addBoth(self.shouldFail, dirnode.NotMutableError,
"bar-ro.mkdir('boffo')")
d.addCallback(lambda res: rootnode.delete("foo"))
@ -218,7 +307,7 @@ class Test(unittest.TestCase):
d.addCallback(lambda res:
rootnode.move_child_to("file4",
self.bar_node_readonly, "boffo"))
d.addBoth(self.shouldFail, vdrive.NotMutableError,
d.addBoth(self.shouldFail, dirnode.NotMutableError,
"mv root/file4 root/bar-ro/boffo")
d.addCallback(lambda res: rootnode.list())
@ -328,26 +417,26 @@ class Encryption(unittest.TestCase):
def test_loopback(self):
key = "k" * 16
data = "This is some plaintext data."
crypttext = vdrive.encrypt(key, data)
plaintext = vdrive.decrypt(key, crypttext)
crypttext = dirnode.encrypt(key, data)
plaintext = dirnode.decrypt(key, crypttext)
self.failUnlessEqual(data, plaintext)
def test_hmac(self):
key = "j" * 16
data = "This is some more plaintext data."
crypttext = vdrive.encrypt(key, data)
crypttext = dirnode.encrypt(key, data)
# flip a bit in the IV
self.failUnlessRaises(vdrive.IntegrityCheckError,
vdrive.decrypt,
self.failUnlessRaises(dirnode.IntegrityCheckError,
dirnode.decrypt,
key, flip_bit(crypttext, 0))
# flip a bit in the crypttext
self.failUnlessRaises(vdrive.IntegrityCheckError,
vdrive.decrypt,
self.failUnlessRaises(dirnode.IntegrityCheckError,
dirnode.decrypt,
key, flip_bit(crypttext, 16))
# flip a bit in the HMAC
self.failUnlessRaises(vdrive.IntegrityCheckError,
vdrive.decrypt,
self.failUnlessRaises(dirnode.IntegrityCheckError,
dirnode.decrypt,
key, flip_bit(crypttext, -1))
plaintext = vdrive.decrypt(key, crypttext)
plaintext = dirnode.decrypt(key, crypttext)
self.failUnlessEqual(data, plaintext)

View File

@ -1,86 +0,0 @@
from twisted.trial import unittest
from allmydata import filetable, uri
from allmydata.util import hashutil
class FileTable(unittest.TestCase):
def test_vdrive_server(self):
basedir = "filetable/FileTable/test_vdrive_server"
vds = filetable.VirtualDriveServer(basedir)
vds.set_furl("myFURL")
root_uri = vds.get_public_root_uri()
self.failUnless(uri.is_dirnode_uri(root_uri))
self.failUnless(uri.is_mutable_dirnode_uri(root_uri))
furl, key = uri.unpack_dirnode_uri(root_uri)
self.failUnlessEqual(furl, "myFURL")
self.failUnlessEqual(len(key), hashutil.KEYLEN)
wk, we, rk, index = hashutil.generate_dirnode_keys_from_writekey(key)
empty_list = vds.list(index)
self.failUnlessEqual(empty_list, [])
vds.set(index, we, "key1", "name1", "write1", "read1")
vds.set(index, we, "key2", "name2", "", "read2")
self.failUnlessRaises(filetable.ChildAlreadyPresentError,
vds.set,
index, we, "key2", "name2", "write2", "read2")
self.failUnlessRaises(filetable.BadWriteEnablerError,
vds.set,
index, "not the write enabler",
"key2", "name2", "write2", "read2")
self.failUnlessEqual(vds.get(index, "key1"),
("write1", "read1"))
self.failUnlessEqual(vds.get(index, "key2"),
("", "read2"))
self.failUnlessRaises(IndexError,
vds.get, index, "key3")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name1", "write1", "read1"),
("name2", "", "read2"),
])
self.failUnlessRaises(filetable.BadWriteEnablerError,
vds.delete,
index, "not the write enabler", "name1")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name1", "write1", "read1"),
("name2", "", "read2"),
])
self.failUnlessRaises(IndexError,
vds.delete,
index, we, "key3")
vds.delete(index, we, "key1")
self.failUnlessEqual(sorted(vds.list(index)),
[ ("name2", "", "read2"),
])
self.failUnlessRaises(IndexError,
vds.get, index, "key1")
self.failUnlessEqual(vds.get(index, "key2"),
("", "read2"))
vds2 = filetable.VirtualDriveServer(basedir)
vds2.set_furl("myFURL")
root_uri2 = vds.get_public_root_uri()
self.failUnless(uri.is_mutable_dirnode_uri(root_uri2))
furl2, key2 = uri.unpack_dirnode_uri(root_uri2)
(wk2, we2, rk2, index2) = \
hashutil.generate_dirnode_keys_from_writekey(key2)
self.failUnlessEqual(sorted(vds2.list(index2)),
[ ("name2", "", "read2"),
])
def test_no_root(self):
basedir = "FileTable/test_no_root"
vds = filetable.VirtualDriveServer(basedir, offer_public_root=False)
vds.set_furl("myFURL")
self.failUnlessRaises(filetable.NoPublicRootError,
vds.get_public_root_uri)

View File

@ -7,7 +7,7 @@ from nevow.static import File as nevow_File # TODO: merge with static.File?
from allmydata.util import idlib
from allmydata.uri import unpack_uri
from allmydata.interfaces import IDownloadTarget, IDirectoryNode, IFileNode
from allmydata.vdrive import FileNode
from allmydata.dirnode import FileNode
from allmydata import upload
from zope.interface import implements, Interface
import urllib