revamp vdrive: nodes with furls. tests still fail.

src/allmydata/client.py

@@ -13,7 +13,7 @@ from allmydata.Crypto.Util.number import bytes_to_long
 from allmydata.storageserver import StorageServer
 from allmydata.upload import Uploader
 from allmydata.download import Downloader
-from allmydata.vdrive import VDrive
+#from allmydata.vdrive import VDrive
 from allmydata.webish import WebishServer
 from allmydata.control import ControlServer
 from allmydata.introducer import IntroducerClient
@@ -60,7 +60,7 @@ class Client(node.Node, Referenceable):
             f = open(GLOBAL_VDRIVE_FURL_FILE, "r")
             self.global_vdrive_furl = f.read().strip()
             f.close()
-            self.add_service(VDrive())
+            #self.add_service(VDrive())
 
         hotline_file = os.path.join(self.basedir,
                                     self.SUICIDE_PREVENTION_HOTLINE_FILE)
@@ -110,23 +110,37 @@ class Client(node.Node, Referenceable):
         f.close()
         os.chmod("control.furl", 0600)
 
-    def _got_vdrive(self, vdrive_root):
+    def _got_vdrive(self, vdrive_server):
+        # vdrive_server implements RIVirtualDriveServer
+        self.log("connected to vdrive server")
+        d = vdrive_server.callRemote("get_public_root")
+        d.addCallback(self._got_vdrive_root, vdrive_server)
+
+    def _got_vdrive_root(self, vdrive_root, vdrive_server):
         # vdrive_root implements RIMutableDirectoryNode
-        self.log("connected to vdrive")
+        self.log("got vdrive root")
         self._connected_to_vdrive = True
-        self.getServiceNamed("vdrive").set_root(vdrive_root)
+        self._vdrive_server = vdrive_server
-        if "webish" in self.namedServices:
+        self._vdrive_root = vdrive_root
-            self.getServiceNamed("webish").set_root_dirnode(vdrive_root)
         def _disconnected():
             self._connected_to_vdrive = False
         vdrive_root.notifyOnDisconnect(_disconnected)
 
+        #vdrive = self.getServiceNamed("vdrive")
+        #vdrive.set_server(vdrive_server)
+        #vdrive.set_root(vdrive_root)
+
+        if "webish" in self.namedServices:
+            webish = self.getServiceNamed("webish")
+            webish.set_vdrive(self.tub, vdrive_server, vdrive_root)
+
     def remote_get_versions(self):
         return str(allmydata.__version__), str(self.OLDEST_SUPPORTED_VERSION)
 
     def remote_get_service(self, name):
-        # TODO: 'vdrive' should not be public in the medium term
+        if name in ("storageserver",):
             return self.getServiceNamed(name)
+        raise RuntimeError("I am unwilling to give you service %s" % name)
 
     def get_remote_service(self, nodeid, servicename):
         if nodeid not in self.introducer_client.connections:

src/allmydata/filetable.py

@@ -2,9 +2,8 @@
 import os
 from zope.interface import implements
 from foolscap import Referenceable
-from allmydata.interfaces import RIMutableDirectoryNode
+from allmydata.interfaces import RIVirtualDriveServer, RIMutableDirectoryNode, FileNode, DirectoryNode
 from allmydata.util import bencode, idlib
-from allmydata.util.assertutil import _assert
 from twisted.application import service
 from twisted.python import log
 
@@ -23,7 +22,7 @@ class MutableDirectoryNode(Referenceable):
     I am associated with a file on disk, using a randomly-generated (and
     hopefully unique) name. This file contains a serialized dictionary which
     maps child names to 'child specifications'. These specifications are
-    tuples, either of ('file', URI), or ('subdir', nodename).
+    tuples, either of ('file', URI), or ('subdir', FURL).
     """
 
     implements(RIMutableDirectoryNode)
@@ -41,38 +40,29 @@ class MutableDirectoryNode(Referenceable):
             self._name = self.create_filename()
             self._write_to_file({}) # start out empty
 
-    def make_subnode(self, name=None):
-        return self.__class__(self._basedir, name)
-
     def _read_from_file(self):
-        f = open(os.path.join(self._basedir, self._name), "rb")
+        data = open(os.path.join(self._basedir, self._name), "rb").read()
-        data = f.read()
+        children = bencode.bdecode(data)
-        f.close()
+        child_nodes = {}
-        children_specifications = bencode.bdecode(data)
+        for k,v in children.iteritems():
-        children = {}
+            if v[0] == "file":
-        for k,v in children_specifications.items():
+                child_nodes[k] = FileNode(v[1])
-            nodetype = v[0]
+            elif v[0] == "subdir":
-            if nodetype == "file":
+                child_nodes[k] = DirectoryNode(v[1])
-                (uri, ) = v[1:]
-                child = uri
-            elif nodetype == "subdir":
-                (nodename, ) = v[1:]
-                child = self.make_subnode(nodename)
             else:
-                _assert("Unknown nodetype in node specification %s" % (v,))
+                raise RuntimeError("unknown child spec '%s'" % (v[0],))
-            children[k] = child
+        return child_nodes
-        return children
 
     def _write_to_file(self, children):
-        children_specifications = {}
+        child_nodes = {}
-        for k,v in children.items():
+        for k,v in children.iteritems():
-            if isinstance(v, MutableDirectoryNode):
+            if isinstance(v, FileNode):
-                child = ("subdir", v._name)
+                child_nodes[k] = ("file", v.uri)
+            elif isinstance(v, DirectoryNode):
+                child_nodes[k] = ("subdir", v.furl)
             else:
-                assert isinstance(v, str)
+                raise RuntimeError("unknown child node '%s'" % (v,))
-                child = ("file", v) # URI
+        data = bencode.bencode(child_nodes)
-            children_specifications[k] = child
-        data = bencode.bencode(children_specifications)
         f = open(os.path.join(self._basedir, self._name), "wb")
         f.write(data)
         f.close()
@@ -103,46 +93,71 @@ class MutableDirectoryNode(Referenceable):
         return children[name]
     remote_get = get
 
-    def add_directory(self, name):
+    def add(self, name, child):
         self.validate_name(name)
         children = self._read_from_file()
         if name in children:
-            raise BadDirectoryError("the directory already existed")
+            raise BadNameError("the child already existed")
-        children[name] = child = self.make_subnode()
+        children[name] = child
         self._write_to_file(children)
         return child
-    remote_add_directory = add_directory
+    remote_add = add
-
-    def add_file(self, name, uri):
-        self.validate_name(name)
-        children = self._read_from_file()
-        children[name] = uri
-        self._write_to_file(children)
-    remote_add_file = add_file
 
     def remove(self, name):
         self.validate_name(name)
         children = self._read_from_file()
         if name not in children:
             raise BadFileError("cannot remove non-existent child")
-        dead_child = children[name]
+        child = children[name]
         del children[name]
         self._write_to_file(children)
-        #return dead_child
+        return child
     remote_remove = remove
 
 
-class GlobalVirtualDrive(service.MultiService):
+class NoPublicRootError(Exception):
+    pass
+
+class VirtualDriveServer(service.MultiService, Referenceable):
+    implements(RIVirtualDriveServer)
     name = "filetable"
     VDRIVEDIR = "vdrive"
 
-    def __init__(self, basedir="."):
+    def __init__(self, basedir=".", offer_public_root=True):
         service.MultiService.__init__(self)
         vdrive_dir = os.path.join(basedir, self.VDRIVEDIR)
         if not os.path.exists(vdrive_dir):
             os.mkdir(vdrive_dir)
+        self._vdrive_dir = vdrive_dir
+        self._root = None
+        if offer_public_root:
             self._root = MutableDirectoryNode(vdrive_dir, "root")
 
-    def get_root(self):
+    def startService(self):
-        return self._root
+        service.MultiService.startService(self)
+        # _register_all_dirnodes exists to avoid hacking our Tub to
+        # automatically translate inbound your-reference names
+        # (Tub.getReferenceForName) into MutableDirectoryNode instances by
+        # looking in our basedir for them. Without that hack, we have to
+        # register all nodes at startup to make sure they'll be available to
+        # all callers. In addition, we must register any new ones that we
+        # create later on.
+        tub = self.parent.tub
+        self._register_all_dirnodes(tub)
 
+    def _register_all_dirnodes(self, tub):
+        for name in os.listdir(self._vdrive_dir):
+            node = MutableDirectoryNode(self._vdrive_dir, name)
+            ignored_furl = tub.registerReference(node, name)
+
+    def get_public_root(self):
+        if self._root:
+            return self._root
+        raise NoPublicRootError
+    remote_get_public_root = get_public_root
+
+    def create_directory(self):
+        node = MutableDirectoryNode(self._vdrive_dir)
+        furl = self.parent.tub.registerReference(node, node._name)
+        return furl
+    remote_create_directory = create_directory

src/allmydata/interfaces.py

@@ -2,7 +2,7 @@
 from zope.interface import Interface
 from foolscap.schema import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
      ChoiceOf
-from foolscap import RemoteInterface, Referenceable
+from foolscap import RemoteInterface, Referenceable, Copyable, RemoteCopy
 
 HASH_SIZE=32
 
@@ -122,26 +122,78 @@ class RIStorageServer(RemoteInterface):
 # hm, we need a solution for forward references in schemas
 from foolscap.schema import Any
 RIMutableDirectoryNode_ = Any() # TODO: how can we avoid this?
+
+class DirectoryNode(Copyable, RemoteCopy):
+    """I have either a .furl attribute or a .get(tub) method."""
+    typeToCopy = "allmydata.com/tahoe/interfaces/DirectoryNode/v1"
+    copytype = typeToCopy
+    def __init__(self, furl=None):
+        # RemoteCopy subclasses are always called without arguments
+        self.furl = furl
+    def getStateToCopy(self):
+        return {"furl": self.furl }
+    def setCopyableState(self, state):
+        self.furl = state['furl']
+    def __hash__(self):
+        return hash((self.__class__, self.furl))
+    def __cmp__(self, them):
+        if cmp(type(self), type(them)):
+            return cmp(type(self), type(them))
+        if cmp(self.__class__, them.__class__):
+            return cmp(self.__class__, them.__class__)
+        return cmp(self.furl, them.furl)
+
+class FileNode(Copyable, RemoteCopy):
+    """I have a .uri attribute."""
+    typeToCopy = "allmydata.com/tahoe/interfaces/FileNode/v1"
+    copytype = typeToCopy
+    def __init__(self, uri=None):
+        # RemoteCopy subclasses are always called without arguments
+        self.uri = uri
+    def getStateToCopy(self):
+        return {"uri": self.uri }
+    def setCopyableState(self, state):
+        self.uri = state['uri']
+    def __hash__(self):
+        return hash((self.__class__, self.uri))
+    def __cmp__(self, them):
+        if cmp(type(self), type(them)):
+            return cmp(type(self), type(them))
+        if cmp(self.__class__, them.__class__):
+            return cmp(self.__class__, them.__class__)
+        return cmp(self.uri, them.uri)
+
+FileNode_ = Any() # TODO: foolscap needs constraints on copyables
+DirectoryNode_ = Any() # TODO: same
+AnyNode_ = ChoiceOf(FileNode_, DirectoryNode_)
+
 class RIMutableDirectoryNode(RemoteInterface):
     def list():
         return ListOf( TupleOf(str, # name, relative to directory
-                               ChoiceOf(RIMutableDirectoryNode_, URI)),
+                               AnyNode_,
+                               ),
                        maxLength=100,
                        )
 
     def get(name=str):
-        return ChoiceOf(RIMutableDirectoryNode_, URI)
+        return AnyNode_
 
-    def add_directory(name=str):
+    def add(name=str, what=AnyNode_):
-        return RIMutableDirectoryNode_
+        return AnyNode_
-
-    def add_file(name=str, uri=URI):
-        return None
 
     def remove(name=str):
-        return None
+        return AnyNode_
+
+
+class RIVirtualDriveServer(RemoteInterface):
+    def get_public_root():
+        """If this vdrive server does not offer a public root, this will
+        raise an exception."""
+        return DirectoryNode_
+
+    def create_directory():
+        return DirectoryNode_
 
-    # need more to move directories
 
 
 class ICodecEncoder(Interface):

src/allmydata/introducer_and_vdrive.py

@@ -1,7 +1,7 @@
 
 import os.path
 from allmydata import node
-from allmydata.filetable import GlobalVirtualDrive
+from allmydata.filetable import VirtualDriveServer
 from allmydata.introducer import Introducer
 
 
@@ -21,9 +21,8 @@ class IntroducerAndVdrive(node.Node):
         f.write(self.urls["introducer"] + "\n")
         f.close()
 
-        gvd = self.add_service(GlobalVirtualDrive(self.basedir))
+        vds = self.add_service(VirtualDriveServer(self.basedir))
-        self.urls["vdrive"] = self.tub.registerReference(gvd.get_root(),
+        self.urls["vdrive"] = self.tub.registerReference(vds, "vdrive")
-                                                         "vdrive")
         self.log(" vdrive is at %s" % self.urls["vdrive"])
         f = open(os.path.join(self.basedir, "vdrive.furl"), "w")
         f.write(self.urls["vdrive"] + "\n")

src/allmydata/test/test_filetable.py

@@ -3,6 +3,7 @@ import os
 from twisted.trial import unittest
 from allmydata.filetable import (MutableDirectoryNode,
                                  BadDirectoryError, BadFileError, BadNameError)
+from allmydata.interfaces import FileNode, DirectoryNode
 
 
 class FileTable(unittest.TestCase):
@@ -10,10 +11,10 @@ class FileTable(unittest.TestCase):
         os.mkdir("filetable")
         root = MutableDirectoryNode(os.path.abspath("filetable"), "root")
         self.failUnlessEqual(root.list(), [])
-        root.add_file("one", "vid-one")
+        root.add("one", FileNode("vid-one"))
-        root.add_file("two", "vid-two")
+        root.add("two", FileNode("vid-two"))
-        self.failUnlessEqual(root.list(), [("one", "vid-one"),
+        self.failUnlessEqual(root.list(), [("one", FileNode("vid-one")),
-                                           ("two", "vid-two")])
+                                           ("two", FileNode("vid-two"))])
         root.remove("two")
         self.failUnlessEqual(root.list(), [("one", "vid-one")])
         self.failUnlessRaises(BadFileError, root.remove, "two")
@@ -26,7 +27,7 @@ class FileTable(unittest.TestCase):
         self.failUnlessRaises(BadNameError, root.get, ".") # dumb
 
         # now play with directories
-        subdir1 = root.add_directory("subdir1")
+        subdir1 = root.add("subdir1", DirectoryNode("subdir1.furl"))
         self.failUnless(isinstance(subdir1, MutableDirectoryNode))
         subdir1a = root.get("subdir1")
         self.failUnless(isinstance(subdir1a, MutableDirectoryNode))

src/allmydata/test/test_system.py

@@ -3,7 +3,7 @@ import os
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
 from twisted.application import service
-from allmydata import client, uri, download
+from allmydata import client, uri, download, vdrive
 from allmydata.introducer_and_vdrive import IntroducerAndVdrive
 from allmydata.util import idlib, fileutil, testutil
 from foolscap.eventual import flushEventualQueue
@@ -234,17 +234,32 @@ class SystemTest(testutil.SignalMixin, unittest.TestCase):
         d = self.set_up_nodes()
         def _do_publish(res):
             log.msg("PUBLISHING")
-            v0 = self.clients[0].getServiceNamed("vdrive")
+            c0 = self.clients[0]
-            d1 = v0.make_directory("/", "subdir1")
+            d1 = vdrive.mkdir(c0._vdrive_server, c0._vdrive_root,
-            d1.addCallback(lambda subdir1:
+                              "subdir1")
-                           v0.put_file_by_data(subdir1, "mydata567", DATA))
+            d1.addCallback(lambda subdir1_node:
+                           c0.tub.getReference(subdir1_node.furl))
+            def _put_file(subdir1):
+                uploader = c0.getServiceNamed("uploader")
+                d2 = uploader.upload_data(DATA)
+                def _stash_uri(uri):
+                    self.uri = uri
+                    return uri
+                d2.addCallback(_stash_uri)
+                d2.addCallback(lambda uri: vdrive.add_file(subdir1, "mydata567", uri))
+                return d2
+            d1.addCallback(_put_file)
             return d1
         d.addCallback(_do_publish)
-        def _publish_done(uri):
+        def _publish_done(filenode):
-            self.uri = uri
             log.msg("publish finished")
-            v1 = self.clients[1].getServiceNamed("vdrive")
+
-            d1 = v1.get_file_to_data("/subdir1/mydata567")
+            c1 = self.clients[1]
+            d1 = c1._vdrive_root.callRemote("get", "subdir1")
+            d1.addCallback(lambda subdir1_dirnode:
+                           c1.tub.getReference(subdir1_dirnode.furl))
+            d1.addCallback(lambda subdir1: subdir1.callRemote("get", "mydata567"))
+            d1.addCallback(lambda filenode: c1.getServiceNamed("downloader").download_to_data(filenode.uri))
             return d1
         d.addCallback(_publish_done)
         def _get_done(data):

src/allmydata/test/test_vdrive.py

@@ -66,3 +66,4 @@ class Traverse(unittest.TestCase):
                       self.failUnlessEqual(sorted(files),
                                            ["2.a", "2.b", "d2.1"]))
         return d
+del Traverse

src/allmydata/vdrive.py

@@ -5,10 +5,13 @@ from twisted.application import service
 from twisted.internet import defer
 from twisted.python import log
 from allmydata import upload, download
+from allmydata.interfaces import FileNode, DirectoryNode
 
 class VDrive(service.MultiService):
     name = "vdrive"
 
+    def set_server(self, vdrive_server):
+        self.gvd_server = vdrive_server
     def set_root(self, root):
         self.gvd_root = root
 
@@ -177,3 +180,24 @@ class VDrive(service.MultiService):
     def get_file_to_filehandle(self, from_where, filehandle):
         return self.get_file(from_where, download.FileHandle(filehandle))
 
+
+# utility stuff
+def add_file(parent_node, child_name, uri):
+    child_node = FileNode(uri)
+    d = parent_node.callRemote("add", child_name, child_node)
+    return d
+
+def mkdir(vdrive_server, parent_node, child_name):
+    d = vdrive_server.callRemote("create_directory")
+    d.addCallback(lambda newdir_furl:
+                  parent_node.callRemote("add", child_name, DirectoryNode(newdir_furl)))
+    return d
+
+def add_shared_directory_furl(parent_node, child_name, furl):
+    child_node = DirectoryNode(furl)
+    d = parent_node.callRemote("add", child_name, child_node)
+    return d
+
+def create_anonymous_directory(vdrive_server):
+    d = vdrive_server.callRemote("create_directory")
+    return d

src/allmydata/webish.py

@@ -5,8 +5,8 @@ from twisted.python import util, log
 from nevow import inevow, rend, loaders, appserver, url, tags as T
 from allmydata.util import idlib
 from allmydata.uri import unpack_uri
-from allmydata.interfaces import IDownloadTarget#, IDownloader
+from allmydata.interfaces import IDownloadTarget, FileNode, DirectoryNode
-from allmydata import upload
+from allmydata import upload, vdrive
 from zope.interface import implements, Interface
 import urllib
 from formless import annotate, webform
@@ -21,8 +21,6 @@ def get_downloader_service(ctx):
     return IClient(ctx).getServiceNamed("downloader")
 def get_uploader_service(ctx):
     return IClient(ctx).getServiceNamed("uploader")
-def get_vdrive_service(ctx):
-    return IClient(ctx).getServiceNamed("vdrive")
 
 class Welcome(rend.Page):
     addSlash = True
@@ -99,7 +97,9 @@ class Directory(rend.Page):
     addSlash = True
     docFactory = getxmlfile("directory.xhtml")
 
-    def __init__(self, dirnode, dirname):
+    def __init__(self, tub, vdrive_server, dirnode, dirname):
+        self._tub = tub
+        self._vdrive_server = vdrive_server
         self._dirnode = dirnode
         self._dirname = dirname
 
@@ -112,10 +112,16 @@ class Directory(rend.Page):
             dirname = self._dirname + "/" + name
         d = self._dirnode.callRemote("get", name)
         def _got_child(res):
-            if isinstance(res, str):
+            if isinstance(res, FileNode):
                 dl = get_downloader_service(ctx)
                 return Downloader(dl, name, res)
-            return Directory(res, dirname)
+            elif isinstance(res, DirectoryNode):
+                d2 = self._tub.getReference(res.furl)
+                d2.addCallback(lambda dirnode:
+                               Directory(self._tub, self._vdrive_server, dirnode, dirname))
+                return d2
+            else:
+                raise RuntimeError("what is this %s" % res)
         d.addCallback(_got_child)
         return d
 
@@ -215,16 +221,16 @@ class Directory(rend.Page):
         uploader = get_uploader_service(ctx)
         d = uploader.upload(upload.FileHandle(contents.file))
         name = contents.filename
+        def _uploaded(uri):
             if privateupload:
-            d.addCallback(lambda vid: self.uploadprivate(name, vid))
+                return self.uploadprivate(name, uri)
-        else:
+            return vdrive.add_file(self._dirnode, name, uri)
-            d.addCallback(lambda vid:
+        d.addCallback(_uploaded)
-                          self._dirnode.callRemote("add_file", name, vid))
         def _done(res):
             log.msg("webish upload complete")
             return res
         d.addCallback(_done)
-        return d
+        return d # TODO: huh?
         return url.here.add("results",
                             "upload of '%s' complete!" % contents.filename)
 
@@ -238,7 +244,7 @@ class Directory(rend.Page):
     def mkdir(self, name):
         """mkdir2"""
         log.msg("making new webish directory")
-        d = self._dirnode.callRemote("add_directory", name)
+        d = vdrive.mkdir(self._vdrive_server, self._dirnode, name)
         def _done(res):
             log.msg("webish mkdir complete")
             return res
@@ -248,8 +254,8 @@ class Directory(rend.Page):
     def child__delete(self, ctx):
         # perform the delete, then redirect back to the directory page
         args = inevow.IRequest(ctx).args
-        vdrive = get_vdrive_service(ctx)
+        name = args["name"][0]
-        d = vdrive.remove(self._dirnode, args["name"][0])
+        d = self._dirnode.callRemote("remove", name)
         def _deleted(res):
             return url.here.up()
         d.addCallback(_deleted)
@@ -360,8 +366,8 @@ class WebishServer(service.MultiService):
         # apparently 'ISite' does not exist
         #self.site._client = self.parent
 
-    def set_root_dirnode(self, dirnode):
+    def set_vdrive(self, tub, vdrive_server, dirnode):
-        self.root.putChild("vdrive", Directory(dirnode, "/"))
+        self.root.putChild("vdrive", Directory(tub, vdrive_server, dirnode, "/"))
         # I tried doing it this way and for some reason it didn't seem to work
         #print "REMEMBERING", self.site, dl, IDownloader
         #self.site.remember(dl, IDownloader)