revamp vdrive: nodes with furls. tests still fail.

This commit is contained in:
Brian Warner 2007-06-14 20:14:34 -07:00
parent 106177a7f2
commit ec77a227be
9 changed files with 230 additions and 103 deletions

View File

@ -13,7 +13,7 @@ from allmydata.Crypto.Util.number import bytes_to_long
from allmydata.storageserver import StorageServer
from allmydata.upload import Uploader
from allmydata.download import Downloader
from allmydata.vdrive import VDrive
#from allmydata.vdrive import VDrive
from allmydata.webish import WebishServer
from allmydata.control import ControlServer
from allmydata.introducer import IntroducerClient
@ -60,7 +60,7 @@ class Client(node.Node, Referenceable):
f = open(GLOBAL_VDRIVE_FURL_FILE, "r")
self.global_vdrive_furl = f.read().strip()
f.close()
self.add_service(VDrive())
#self.add_service(VDrive())
hotline_file = os.path.join(self.basedir,
self.SUICIDE_PREVENTION_HOTLINE_FILE)
@ -110,23 +110,37 @@ class Client(node.Node, Referenceable):
f.close()
os.chmod("control.furl", 0600)
def _got_vdrive(self, vdrive_root):
def _got_vdrive(self, vdrive_server):
# vdrive_server implements RIVirtualDriveServer
self.log("connected to vdrive server")
d = vdrive_server.callRemote("get_public_root")
d.addCallback(self._got_vdrive_root, vdrive_server)
def _got_vdrive_root(self, vdrive_root, vdrive_server):
# vdrive_root implements RIMutableDirectoryNode
self.log("connected to vdrive")
self.log("got vdrive root")
self._connected_to_vdrive = True
self.getServiceNamed("vdrive").set_root(vdrive_root)
if "webish" in self.namedServices:
self.getServiceNamed("webish").set_root_dirnode(vdrive_root)
self._vdrive_server = vdrive_server
self._vdrive_root = vdrive_root
def _disconnected():
self._connected_to_vdrive = False
vdrive_root.notifyOnDisconnect(_disconnected)
#vdrive = self.getServiceNamed("vdrive")
#vdrive.set_server(vdrive_server)
#vdrive.set_root(vdrive_root)
if "webish" in self.namedServices:
webish = self.getServiceNamed("webish")
webish.set_vdrive(self.tub, vdrive_server, vdrive_root)
def remote_get_versions(self):
return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION)
def remote_get_service(self, name):
# TODO: 'vdrive' should not be public in the medium term
return self.getServiceNamed(name)
if name in ("storageserver",):
return self.getServiceNamed(name)
raise RuntimeError("I am unwilling to give you service %s" % name)
def get_remote_service(self, nodeid, servicename):
if nodeid not in self.introducer_client.connections:

View File

@ -2,9 +2,8 @@
import os
from zope.interface import implements
from foolscap import Referenceable
from allmydata.interfaces import RIMutableDirectoryNode
from allmydata.interfaces import RIVirtualDriveServer, RIMutableDirectoryNode, FileNode, DirectoryNode
from allmydata.util import bencode, idlib
from allmydata.util.assertutil import _assert
from twisted.application import service
from twisted.python import log
@ -23,7 +22,7 @@ class MutableDirectoryNode(Referenceable):
I am associated with a file on disk, using a randomly-generated (and
hopefully unique) name. This file contains a serialized dictionary which
maps child names to 'child specifications'. These specifications are
tuples, either of ('file', URI), or ('subdir', nodename).
tuples, either of ('file', URI), or ('subdir', FURL).
"""
implements(RIMutableDirectoryNode)
@ -41,38 +40,29 @@ class MutableDirectoryNode(Referenceable):
self._name = self.create_filename()
self._write_to_file({}) # start out empty
def make_subnode(self, name=None):
return self.__class__(self._basedir, name)
def _read_from_file(self):
f = open(os.path.join(self._basedir, self._name), "rb")
data = f.read()
f.close()
children_specifications = bencode.bdecode(data)
children = {}
for k,v in children_specifications.items():
nodetype = v[0]
if nodetype == "file":
(uri, ) = v[1:]
child = uri
elif nodetype == "subdir":
(nodename, ) = v[1:]
child = self.make_subnode(nodename)
data = open(os.path.join(self._basedir, self._name), "rb").read()
children = bencode.bdecode(data)
child_nodes = {}
for k,v in children.iteritems():
if v[0] == "file":
child_nodes[k] = FileNode(v[1])
elif v[0] == "subdir":
child_nodes[k] = DirectoryNode(v[1])
else:
_assert("Unknown nodetype in node specification %s" % (v,))
children[k] = child
return children
raise RuntimeError("unknown child spec '%s'" % (v[0],))
return child_nodes
def _write_to_file(self, children):
children_specifications = {}
for k,v in children.items():
if isinstance(v, MutableDirectoryNode):
child = ("subdir", v._name)
child_nodes = {}
for k,v in children.iteritems():
if isinstance(v, FileNode):
child_nodes[k] = ("file", v.uri)
elif isinstance(v, DirectoryNode):
child_nodes[k] = ("subdir", v.furl)
else:
assert isinstance(v, str)
child = ("file", v) # URI
children_specifications[k] = child
data = bencode.bencode(children_specifications)
raise RuntimeError("unknown child node '%s'" % (v,))
data = bencode.bencode(child_nodes)
f = open(os.path.join(self._basedir, self._name), "wb")
f.write(data)
f.close()
@ -103,46 +93,71 @@ class MutableDirectoryNode(Referenceable):
return children[name]
remote_get = get
def add_directory(self, name):
def add(self, name, child):
self.validate_name(name)
children = self._read_from_file()
if name in children:
raise BadDirectoryError("the directory already existed")
children[name] = child = self.make_subnode()
raise BadNameError("the child already existed")
children[name] = child
self._write_to_file(children)
return child
remote_add_directory = add_directory
def add_file(self, name, uri):
self.validate_name(name)
children = self._read_from_file()
children[name] = uri
self._write_to_file(children)
remote_add_file = add_file
remote_add = add
def remove(self, name):
self.validate_name(name)
children = self._read_from_file()
if name not in children:
raise BadFileError("cannot remove non-existent child")
dead_child = children[name]
child = children[name]
del children[name]
self._write_to_file(children)
#return dead_child
return child
remote_remove = remove
class GlobalVirtualDrive(service.MultiService):
class NoPublicRootError(Exception):
pass
class VirtualDriveServer(service.MultiService, Referenceable):
implements(RIVirtualDriveServer)
name = "filetable"
VDRIVEDIR = "vdrive"
def __init__(self, basedir="."):
def __init__(self, basedir=".", offer_public_root=True):
service.MultiService.__init__(self)
vdrive_dir = os.path.join(basedir, self.VDRIVEDIR)
if not os.path.exists(vdrive_dir):
os.mkdir(vdrive_dir)
self._root = MutableDirectoryNode(vdrive_dir, "root")
self._vdrive_dir = vdrive_dir
self._root = None
if offer_public_root:
self._root = MutableDirectoryNode(vdrive_dir, "root")
def get_root(self):
return self._root
def startService(self):
service.MultiService.startService(self)
# _register_all_dirnodes exists to avoid hacking our Tub to
# automatically translate inbound your-reference names
# (Tub.getReferenceForName) into MutableDirectoryNode instances by
# looking in our basedir for them. Without that hack, we have to
# register all nodes at startup to make sure they'll be available to
# all callers. In addition, we must register any new ones that we
# create later on.
tub = self.parent.tub
self._register_all_dirnodes(tub)
def _register_all_dirnodes(self, tub):
for name in os.listdir(self._vdrive_dir):
node = MutableDirectoryNode(self._vdrive_dir, name)
ignored_furl = tub.registerReference(node, name)
def get_public_root(self):
if self._root:
return self._root
raise NoPublicRootError
remote_get_public_root = get_public_root
def create_directory(self):
node = MutableDirectoryNode(self._vdrive_dir)
furl = self.parent.tub.registerReference(node, node._name)
return furl
remote_create_directory = create_directory

View File

@ -2,7 +2,7 @@
from zope.interface import Interface
from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
ChoiceOf
from foolscap import RemoteInterface, Referenceable
from foolscap import RemoteInterface, Referenceable, Copyable, RemoteCopy
HASH_SIZE=32
@ -122,26 +122,78 @@ class RIStorageServer(RemoteInterface):
# hm, we need a solution for forward references in schemas
from foolscap.schema import Any
RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this?
class DirectoryNode(Copyable, RemoteCopy):
"""I have either a .furl attribute or a .get(tub) method."""
typeToCopy = "allmydata.com/tahoe/interfaces/DirectoryNode/v1"
copytype = typeToCopy
def __init__(self, furl=None):
# RemoteCopy subclasses are always called without arguments
self.furl = furl
def getStateToCopy(self):
return {"furl": self.furl }
def setCopyableState(self, state):
self.furl = state['furl']
def __hash__(self):
return hash((self.__class__, self.furl))
def __cmp__(self, them):
if cmp(type(self), type(them)):
return cmp(type(self), type(them))
if cmp(self.__class__, them.__class__):
return cmp(self.__class__, them.__class__)
return cmp(self.furl, them.furl)
class FileNode(Copyable, RemoteCopy):
"""I have a .uri attribute."""
typeToCopy = "allmydata.com/tahoe/interfaces/FileNode/v1"
copytype = typeToCopy
def __init__(self, uri=None):
# RemoteCopy subclasses are always called without arguments
self.uri = uri
def getStateToCopy(self):
return {"uri": self.uri }
def setCopyableState(self, state):
self.uri = state['uri']
def __hash__(self):
return hash((self.__class__, self.uri))
def __cmp__(self, them):
if cmp(type(self), type(them)):
return cmp(type(self), type(them))
if cmp(self.__class__, them.__class__):
return cmp(self.__class__, them.__class__)
return cmp(self.uri, them.uri)
FileNode_ = Any() # TODO: foolscap needs constraints on copyables
DirectoryNode_ = Any() # TODO: same
AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_)
class RIMutableDirectoryNode(RemoteInterface):
def list():
return ListOf( TupleOf(str, # name, relative to directory
ChoiceOf(RIMutableDirectoryNode_, URI)),
AnyNode_,
),
maxLength=100,
)
def get(name=str):
return ChoiceOf(RIMutableDirectoryNode_, URI)
return AnyNode_
def add_directory(name=str):
return RIMutableDirectoryNode_
def add_file(name=str, uri=URI):
return None
def add(name=str, what=AnyNode_):
return AnyNode_
def remove(name=str):
return None
return AnyNode_
class RIVirtualDriveServer(RemoteInterface):
def get_public_root():
"""If this vdrive server does not offer a public root, this will
raise an exception."""
return DirectoryNode_
def create_directory():
return DirectoryNode_
# need more to move directories
class ICodecEncoder(Interface):

View File

@ -1,7 +1,7 @@
import os.path
from allmydata import node
from allmydata.filetable import GlobalVirtualDrive
from allmydata.filetable import VirtualDriveServer
from allmydata.introducer import Introducer
@ -21,9 +21,8 @@ class IntroducerAndVdrive(node.Node):
f.write(self.urls["introducer"] + "\n")
f.close()
gvd = self.add_service(GlobalVirtualDrive(self.basedir))
self.urls["vdrive"] = self.tub.registerReference(gvd.get_root(),
"vdrive")
vds = self.add_service(VirtualDriveServer(self.basedir))
self.urls["vdrive"] = self.tub.registerReference(vds, "vdrive")
self.log(" vdrive is at %s" % self.urls["vdrive"])
f = open(os.path.join(self.basedir, "vdrive.furl"), "w")
f.write(self.urls["vdrive"] + "\n")

View File

@ -3,6 +3,7 @@ import os
from twisted.trial import unittest
from allmydata.filetable import (MutableDirectoryNode,
BadDirectoryError, BadFileError, BadNameError)
from allmydata.interfaces import FileNode, DirectoryNode
class FileTable(unittest.TestCase):
@ -10,10 +11,10 @@ class FileTable(unittest.TestCase):
os.mkdir("filetable")
root = MutableDirectoryNode(os.path.abspath("filetable"), "root")
self.failUnlessEqual(root.list(), [])
root.add_file("one", "vid-one")
root.add_file("two", "vid-two")
self.failUnlessEqual(root.list(), [("one", "vid-one"),
("two", "vid-two")])
root.add("one", FileNode("vid-one"))
root.add("two", FileNode("vid-two"))
self.failUnlessEqual(root.list(), [("one", FileNode("vid-one")),
("two", FileNode("vid-two"))])
root.remove("two")
self.failUnlessEqual(root.list(), [("one", "vid-one")])
self.failUnlessRaises(BadFileError, root.remove, "two")
@ -26,7 +27,7 @@ class FileTable(unittest.TestCase):
self.failUnlessRaises(BadNameError, root.get, ".") # dumb
# now play with directories
subdir1 = root.add_directory("subdir1")
subdir1 = root.add("subdir1", DirectoryNode("subdir1.furl"))
self.failUnless(isinstance(subdir1, MutableDirectoryNode))
subdir1a = root.get("subdir1")
self.failUnless(isinstance(subdir1a, MutableDirectoryNode))

View File

@ -3,7 +3,7 @@ import os
from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.application import service
from allmydata import client, uri, download
from allmydata import client, uri, download, vdrive
from allmydata.introducer_and_vdrive import IntroducerAndVdrive
from allmydata.util import idlib, fileutil, testutil
from foolscap.eventual import flushEventualQueue
@ -234,17 +234,32 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase):
d = self.set_up_nodes()
def _do_publish(res):
log.msg("PUBLISHING")
v0 = self.clients[0].getServiceNamed("vdrive")
d1 = v0.make_directory("/", "subdir1")
d1.addCallback(lambda subdir1:
v0.put_file_by_data(subdir1, "mydata567", DATA))
c0 = self.clients[0]
d1 = vdrive.mkdir(c0._vdrive_server, c0._vdrive_root,
"subdir1")
d1.addCallback(lambda subdir1_node:
c0.tub.getReference(subdir1_node.furl))
def _put_file(subdir1):
uploader = c0.getServiceNamed("uploader")
d2 = uploader.upload_data(DATA)
def _stash_uri(uri):
self.uri = uri
return uri
d2.addCallback(_stash_uri)
d2.addCallback(lambda uri: vdrive.add_file(subdir1, "mydata567", uri))
return d2
d1.addCallback(_put_file)
return d1
d.addCallback(_do_publish)
def _publish_done(uri):
self.uri = uri
def _publish_done(filenode):
log.msg("publish finished")
v1 = self.clients[1].getServiceNamed("vdrive")
d1 = v1.get_file_to_data("/subdir1/mydata567")
c1 = self.clients[1]
d1 = c1._vdrive_root.callRemote("get", "subdir1")
d1.addCallback(lambda subdir1_dirnode:
c1.tub.getReference(subdir1_dirnode.furl))
d1.addCallback(lambda subdir1: subdir1.callRemote("get", "mydata567"))
d1.addCallback(lambda filenode: c1.getServiceNamed("downloader").download_to_data(filenode.uri))
return d1
d.addCallback(_publish_done)
def _get_done(data):

View File

@ -66,3 +66,4 @@ class Traverse(unittest.TestCase):
self.failUnlessEqual(sorted(files),
["2.a", "2.b", "d2.1"]))
return d
del Traverse

View File

@ -5,10 +5,13 @@ from twisted.application import service
from twisted.internet import defer
from twisted.python import log
from allmydata import upload, download
from allmydata.interfaces import FileNode, DirectoryNode
class VDrive(service.MultiService):
name = "vdrive"
def set_server(self, vdrive_server):
self.gvd_server = vdrive_server
def set_root(self, root):
self.gvd_root = root
@ -177,3 +180,24 @@ class VDrive(service.MultiService):
def get_file_to_filehandle(self, from_where, filehandle):
return self.get_file(from_where, download.FileHandle(filehandle))
# utility stuff
def add_file(parent_node, child_name, uri):
child_node = FileNode(uri)
d = parent_node.callRemote("add", child_name, child_node)
return d
def mkdir(vdrive_server, parent_node, child_name):
d = vdrive_server.callRemote("create_directory")
d.addCallback(lambda newdir_furl:
parent_node.callRemote("add", child_name, DirectoryNode(newdir_furl)))
return d
def add_shared_directory_furl(parent_node, child_name, furl):
child_node = DirectoryNode(furl)
d = parent_node.callRemote("add", child_name, child_node)
return d
def create_anonymous_directory(vdrive_server):
d = vdrive_server.callRemote("create_directory")
return d

View File

@ -5,8 +5,8 @@ from twisted.python import util, log
from nevow import inevow, rend, loaders, appserver, url, tags as T
from allmydata.util import idlib
from allmydata.uri import unpack_uri
from allmydata.interfaces import IDownloadTarget#, IDownloader
from allmydata import upload
from allmydata.interfaces import IDownloadTarget, FileNode, DirectoryNode
from allmydata import upload, vdrive
from zope.interface import implements, Interface
import urllib
from formless import annotate, webform
@ -21,8 +21,6 @@ def get_downloader_service(ctx):
return IClient(ctx).getServiceNamed("downloader")
def get_uploader_service(ctx):
return IClient(ctx).getServiceNamed("uploader")
def get_vdrive_service(ctx):
return IClient(ctx).getServiceNamed("vdrive")
class Welcome(rend.Page):
addSlash = True
@ -99,7 +97,9 @@ class Directory(rend.Page):
addSlash = True
docFactory = getxmlfile("directory.xhtml")
def __init__(self, dirnode, dirname):
def __init__(self, tub, vdrive_server, dirnode, dirname):
self._tub = tub
self._vdrive_server = vdrive_server
self._dirnode = dirnode
self._dirname = dirname
@ -112,10 +112,16 @@ class Directory(rend.Page):
dirname = self._dirname + "/" + name
d = self._dirnode.callRemote("get", name)
def _got_child(res):
if isinstance(res, str):
if isinstance(res, FileNode):
dl = get_downloader_service(ctx)
return Downloader(dl, name, res)
return Directory(res, dirname)
elif isinstance(res, DirectoryNode):
d2 = self._tub.getReference(res.furl)
d2.addCallback(lambda dirnode:
Directory(self._tub, self._vdrive_server, dirnode, dirname))
return d2
else:
raise RuntimeError("what is this %s" % res)
d.addCallback(_got_child)
return d
@ -215,16 +221,16 @@ class Directory(rend.Page):
uploader = get_uploader_service(ctx)
d = uploader.upload(upload.FileHandle(contents.file))
name = contents.filename
if privateupload:
d.addCallback(lambda vid: self.uploadprivate(name, vid))
else:
d.addCallback(lambda vid:
self._dirnode.callRemote("add_file", name, vid))
def _uploaded(uri):
if privateupload:
return self.uploadprivate(name, uri)
return vdrive.add_file(self._dirnode, name, uri)
d.addCallback(_uploaded)
def _done(res):
log.msg("webish upload complete")
return res
d.addCallback(_done)
return d
return d # TODO: huh?
return url.here.add("results",
"upload of '%s' complete!" % contents.filename)
@ -238,7 +244,7 @@ class Directory(rend.Page):
def mkdir(self, name):
"""mkdir2"""
log.msg("making new webish directory")
d = self._dirnode.callRemote("add_directory", name)
d = vdrive.mkdir(self._vdrive_server, self._dirnode, name)
def _done(res):
log.msg("webish mkdir complete")
return res
@ -248,8 +254,8 @@ class Directory(rend.Page):
def child__delete(self, ctx):
# perform the delete, then redirect back to the directory page
args = inevow.IRequest(ctx).args
vdrive = get_vdrive_service(ctx)
d = vdrive.remove(self._dirnode, args["name"][0])
name = args["name"][0]
d = self._dirnode.callRemote("remove", name)
def _deleted(res):
return url.here.up()
d.addCallback(_deleted)
@ -360,8 +366,8 @@ class WebishServer(service.MultiService):
# apparently 'ISite' does not exist
#self.site._client = self.parent
def set_root_dirnode(self, dirnode):
self.root.putChild("vdrive", Directory(dirnode, "/"))
def set_vdrive(self, tub, vdrive_server, dirnode):
self.root.putChild("vdrive", Directory(tub, vdrive_server, dirnode, "/"))
# I tried doing it this way and for some reason it didn't seem to work
#print "REMEMBERING", self.site, dl, IDownloader
#self.site.remember(dl, IDownloader)