more filetree hacking, still too early to test

src/allmydata/filetree/basenode.py

@@ -2,15 +2,17 @@
 from zope.interface import implements
 from allmydata.filetree.interfaces import INode
 
-class BaseURINode(object):
+class BaseDataNode(object):
     implements(INode)
     prefix = None # must be set by subclass
 
-    def is_directory(self):
+    def get_base_data(self):
-        return False
+        raise NotImplementedError # must be provided by subclass
+    def set_base_data(self, data):
+        raise NotImplementedError # must be provided by subclass
     def serialize_node(self):
-        return "%s:%s" % (self.prefix, self.uri)
+        return "%s:%s" % (self.prefix, self.get_base_data())
     def populate_node(self, data, node_maker):
         assert data.startswith(self.prefix + ":")
-        self.uri = data[len(self.prefix)+1:]
+        self.set_base_data(data[len(self.prefix)+1:])
 

src/allmydata/filetree/directory.py

@@ -5,7 +5,7 @@ from allmydata.filetree.interfaces import (
     ICHKDirectoryNode, ISSKDirectoryNode,
     NoSuchChildError,
     )
-from allmydata.filetree.basenode import BaseURINode
+from allmydata.filetree.basenode import BaseDataNode
 from allmydata import download
 from allmydata.util import bencode
 
@@ -44,9 +44,6 @@ class SubTreeNode:
 #        # then self.child_specifications["foo.jpg"] = ("CHKFILE","fooURI")
 #        self.child_specifications = {}
 
-    def is_directory(self):
-        return True
-
     def list(self):
         return sorted(self.children.keys())
 
@@ -179,10 +176,15 @@ class _DirectorySubTree(object):
                 break
         return (found_path, node, remaining_path)
 
-class CHKDirectorySubTreeNode(BaseURINode):
+class CHKDirectorySubTreeNode(BaseDataNode):
     implements(ICHKDirectoryNode)
     prefix = "CHKDirectory"
 
+    def get_base_data(self):
+        return self.uri
+    def set_base_data(self, data):
+        self.uri = data
+
     def get_uri(self):
         return self.uri
 
@@ -190,9 +192,6 @@ class CHKDirectorySubTreeNode(BaseURINode):
 class CHKDirectorySubTree(_DirectorySubTree):
     # maybe mutable, maybe not
 
-    def mutation_affects_parent(self):
-        return True
-
     def set_uri(self, uri):
         self.old_uri = uri
 
@@ -209,6 +208,7 @@ class CHKDirectorySubTree(_DirectorySubTree):
         self.serialize_to_file(f)
         f.close()
         boxname = work_queue.create_boxname()
+        # mutation affects our parent
         work_queue.add_upload_chk(filename, boxname)
         work_queue.add_delete_tempfile(filename)
         work_queue.add_retain_uri_from_box(boxname)
@@ -225,8 +225,6 @@ class SSKDirectorySubTreeNode(object):
     implements(INode, ISSKDirectoryNode)
     prefix = "SSKDirectory"
 
-    def is_directory(self):
-        return False
     def serialize_node(self):
         data = (self.read_cap, self.write_cap)
         return "%s:%s" % (self.prefix, bencode.bencode(data))
@@ -248,9 +246,6 @@ class SSKDirectorySubTree(_DirectorySubTree):
         self.version = 0
         # TODO: populate
 
-    def mutation_affects_parent(self):
-        return False
-
     def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
         node = ISSKDirectoryNode(node)
         self.read_capability = node.get_read_capability()
@@ -268,6 +263,7 @@ class SSKDirectorySubTree(_DirectorySubTree):
         f, filename = work_queue.create_tempfile(".sskdir")
         self.serialize_to_file(f)
         f.close()
+        # mutation does not affect our parent
         work_queue.add_upload_ssk(filename, self.write_capability,
                                   self.version)
         self.version = self.version + 1

src/allmydata/filetree/file.py

@@ -1,13 +1,18 @@
 
 from zope.interface import implements
 from allmydata.filetree.interfaces import INode, IFileNode
-from allmydata.filetree.basenode import BaseURINode
+from allmydata.filetree.basenode import BaseDataNode
 from allmydata.util import bencode
 
-class CHKFileNode(BaseURINode):
+class CHKFileNode(BaseDataNode):
     implements(IFileNode)
     prefix = "CHKFile"
 
+    def get_base_data(self):
+        return self.uri
+    def set_base_data(self, data):
+        self.uri = data
+
     def get_uri(self):
         return self.uri
 
@@ -15,8 +20,6 @@ class SSKFileNode(object):
     implements(INode, IFileNode)
     prefix = "SSKFile"
 
-    def is_directory(self):
-        return False
     def serialize_node(self):
         data = (self.read_cap, self.write_cap)
         return "%s:%s" % (self.prefix, bencode.bencode(data))

src/allmydata/filetree/interfaces.py

@@ -4,17 +4,16 @@ from zope.interface import Interface
 class INode(Interface):
     """This is some sort of retrievable node. All objects which implement
     other I*Node interfaces also implement this one."""
-    def is_directory():
-        """Return True if this node is an internal directory node."""
     def serialize_node():
         """Return a data structure which contains enough information to build
-        this node again in the future (by calling vdrive.make_node(). For
+        this node again in the future (by calling
-        IDirectoryNodes, this will be a list. For all other nodes this will
+        vdrive.make_node_from_serialized(). For IDirectoryNodes, this will be
-        be a string."""
+        a list. For all other nodes this will be a string."""
     def populate_node(data, node_maker):
-        """vdrive.make_node() will first use the prefix inside 'data' to
+        """vdrive.make_node_from_serialized() will first use the prefix
-        decide what kind of Node to create. It will then call this function
+        inside 'data' to decide what kind of Node to create. It will then
-        to populate the new Node from the data returned by serialize_node."""
+        call this function to populate the new Node from the data returned by
+        serialize_node."""
 
 class IFileNode(Interface):
     """This is a file which can be retrieved."""
@@ -83,12 +82,6 @@ class ISubTree(Interface):
         calls).
         """
 
-    def populate_from_data(data):
-        """Used internally by populate_from_node. This is called with a
-        sequence of bytes that describes the contents of the subtree,
-        probably a bencoded tuple or s-expression. Returns self.
-        """
-
     def is_mutable():
         """This returns True if we have the ability to modify this subtree.
         If this returns True, this reference may be adapted to
@@ -194,7 +187,7 @@ class IVirtualDrive(Interface):
 
     # internal methods
 
-    def make_node(serialized):
+    def make_node_from_serialized(serialized):
         """Given a string produced by original_node.serialize_node(), produce
         an equivalent node.
         """

src/allmydata/filetree/redirect.py

@@ -1,98 +1,183 @@
 
+from cStringIO import StringIO
+from zope.interface import implements
+from twisted.internet import defer
+
+from allmydata.filetree.interfaces import ISubTree
+from allmydata.filetree.basenode import BaseDataNode
 from allmydata.util import bencode
 
-class LocalFileRedirection(object):
+class LocalFileRedirectionNode(BaseDataNode):
+    prefix = "LocalFileRedirection"
+
+    def new(self, handle):
+        self.handle = handle
+
+    def get_base_data(self):
+        return self.handle
+    def set_base_data(self, data):
+        self.handle = data
+
+class _BaseRedirection(object):
+    implements(ISubTree)
+
+    def new(self, child_node):
+        self.child_node = child_node
+
+    def get_node_for_path(self, path):
+        return ([], self.child_node, path)
+
+    def serialize_subtree_to_file(self, f):
+        return self.child_node.serialize_node()
+
+    def _populate_from_data(self, data, node_maker):
+        self.child_node = node_maker(data)
+        return self
+
+class LocalFileRedirection(_BaseRedirection):
     stype = "LocalFileRedirection"
 
-    def populate_from_specification(self, spec, parent_is_mutable, downloader):
+    def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
         # return a Deferred that fires (with self) when this node is ready
         # for use
 
-        (stype, filename) = spec
+        assert isinstance(node, LocalFileRedirectionNode)
-        assert stype == self.stype
+        self.filename = node.handle
-        #filename = spec.get_filename()
+        # there is a local file which contains a bencoded serialized subtree
-        # there is a local file which contains a bencoded serialized
+        # specification.
-        # subtree specification.
 
-        # TODO: will this enable outsiders to cause us to read from
+        # TODO: will this enable outsiders to cause us to read from arbitrary
-        # arbitrary files? Think about this.
+        # files? Think about this. It is probably a good idea to restrict the
-        f = open(filename, "rb")
+        # filename to be a single component, and to always put them in a
+        # well-known directory that contains nothing else, and maybe make
+        # them unguessable.
+        f = open(self.filename, "rb")
         data = f.read()
         f.close()
         # note: we don't cache the contents of the file. TODO: consider
         # doing this based upon mtime. It is important that we be able to
         # notice if the file has been changed.
+        return defer.succeed(self._populate_from_data(data, node_maker))
 
-        return self.populate_from_data(data)
+    def is_mutable(self):
+        return True
 
-    def populate_from_data(self, data):
+    def update(self, prepath, workqueue):
-        # data is a subtree specification for our one child
+        f = open(self.filename, "wb")
-        self.child_spec = bencode.bdecode(data)
+        self.serialize_subtree_to_file(f)
-        return self
+        f.close()
 
-class QueenRedirection(object):
-    stype = "QueenRedirection"
 
-    def populate_from_specification(self, spec, parent_is_mutable, downloader):
+class QueenRedirectionNode(LocalFileRedirectionNode):
-        # this specifies a handle for which the Queen maintains a
+    prefix = "QueenRedirection"
-        # serialized subtree specification.
+
-        (stype, handle) = spec
+class QueenRedirection(_BaseRedirection):
+    style = "QueenRedirection"
+
+    def new(self, handle):
+        self.handle = handle
+
+    def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
+        # this specifies a handle for which the Queen maintains a serialized
+        # subtree specification.
+        assert isinstance(node, QueenRedirectionNode)
+        self.handle = node.handle
 
         # TODO: queen?
-        d = self._queen.callRemote("lookup_handle", handle)
+        d = self._queen.callRemote("lookup_handle", self.handle)
-        d.addCallback(self.populate_from_data)
+        d.addCallback(self._populate_from_data, node_maker)
         return d
 
-    def populate_from_data(self, data):
+    def is_mutable(self):
-        self.child_spec = bencode.bdecode(data)
+        return True # TODO: maybe, maybe not
-        return self
 
-class QueenOrLocalFileRedirection(object):
+    def update(self, prepath, workqueue):
+        f = StringIO()
+        self.serialize_subtree_to_file(f)
+        d = self._queen.callRemote("set_handle", self.handle, f.getvalue())
+        return d
+
+class QueenOrLocalFileRedirectionNode(LocalFileRedirectionNode):
+    prefix = "QueenOrLocalFileRedirection"
+
+class QueenOrLocalFileRedirection(_BaseRedirection):
     stype = "QueenOrLocalFileRedirection"
 
-    def populate_from_specification(self, spec, parent_is_mutable, downloader):
+    def new(self, handle, child_node):
+        self.handle = handle
+        self.version = 0
+        self.child_node = child_node
+        # TODO
+
+    def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
         # there is a local file which contains a bencoded serialized
         # subtree specification. The queen also has a copy. Whomever has
         # the higher version number wins.
-        (stype, filename, handle) = spec
+        assert isinstance(node, QueenOrLocalFileRedirectionNode)
+        self.filename = self.handle = node.handle
 
-        f = open(filename, "rb")
+        f = open(self.filename, "rb")
         #local_version, local_data = bencode.bdecode(f.read())
         local_version_and_data = f.read()
         f.close()
 
         # TODO: queen?
         # TODO: pubsub so we can cache the queen's results
-        d = self._queen.callRemote("lookup_handle", handle)
+        d = self._queen.callRemote("lookup_handle", self.handle)
-        d.addCallback(self._choose_winner, local_version_and_data)
+        d.addCallback(self._choose_winner, local_version_and_data, node_maker)
         return d
 
-    def _choose_winner(self, queen_version_and_data, local_version_and_data):
+    def _choose_winner(self, queen_version_and_data, local_version_and_data, node_maker):
         queen_version, queen_data = bencode.bdecode(queen_version_and_data)
         local_version, local_data = bencode.bdecode(local_version_and_data)
         if queen_version > local_version:
             data = queen_data
+            self.version = queen_version
         else:
             data = local_data
-        return self.populate_from_data(data)
+            self.version = local_version
-
-    def populate_from_data(self, data):
         # NOTE: two layers of bencoding here, TODO
-        self.child_spec = bencode.bdecode(data)
+        return self._populate_from_data(data, node_maker)
-        return self
 
-class HTTPRedirection(object):
+    def is_mutable(self):
-    stype = "HTTPRedirection"
+        return True
 
-    def populate_from_specification(self, spec, parent_is_mutable, downloader):
+    def update(self, prepath, workqueue):
-        # this specifies a URL at which there is a bencoded serialized
+        self.version += 1
-        # subtree specification.
+        f = StringIO()
-        (stype, url) = spec
+        self.serialize_subtree_to_file(f)
-        from twisted.web import client
+        version_and_data = bencode.bencode((self.version, f.getvalue()))
-        d = client.getPage(url)
+        f = open(self.filename, "wb")
-        d.addCallback(self.populate_from_data)
+        f.write(version_and_data)
+        f.close()
+        d = self._queen.callRemote("set_handle", self.handle, version_and_data)
         return d
 
-    def populate_from_data(self, data):
+class HTTPRedirectionNode(BaseDataNode):
-        self.child_spec = bencode.bdecode(data)
+    prefix = "HTTPRedirection"
-        return self
+
+    def new(self, url):
+        self.url = url
+
+    def get_base_data(self):
+        return self.url
+    def set_base_data(self, data):
+        self.url = data
+
+class HTTPRedirection(_BaseRedirection):
+    stype = "HTTPRedirection"
+
+    def populate_from_node(self, node, parent_is_mutable, node_maker, downloader):
+        # this specifies a URL at which there is a bencoded serialized
+        # subtree specification.
+        assert isinstance(node, HTTPRedirectionNode)
+        from twisted.web import client
+        d = client.getPage(node.url)
+        d.addCallback(self._populate_from_data, node_maker)
+        return d
+
+    def is_mutable(self):
+        return False
+
+    def update(self, prepath, workqueue):
+        raise RuntimeError("HTTPRedirection is not mutable")

src/allmydata/filetree/vdrive.py

@@ -8,6 +8,9 @@ from allmydata.filetree.interfaces import (
     )
 from allmydata.upload import IUploadable
 
+# this list is used by VirtualDrive.make_node_from_serialized() to convert
+# node specification strings (found inside the serialized form of subtrees)
+# into Nodes (which live in the in-RAM form of subtrees).
 all_node_types = [
     directory.CHKDirectorySubTreeNode,
     directory.SSKDirectorySubTreeNode,
@@ -57,7 +60,7 @@ class VirtualDrive(object):
 
     def _get_closest_node_2(self, res, parent_is_mutable):
         (found_path, node, remaining_path) = res
-        if node.is_directory():
+        if IDirectoryNode.providedBy(node):
             # traversal done
             return (node, remaining_path)
         # otherwise, we must open and recurse into a new subtree
@@ -125,7 +128,7 @@ class VirtualDrive(object):
         return d
 
     # these are called when loading and creating nodes
-    def make_node(self, serialized):
+    def make_node_from_serialized(self, serialized):
         # this turns a string into an INode, which contains information about
         # the file or directory (like a URI), but does not contain the actual
         # contents. An IOpener can be used later to retrieve the contents
@@ -139,7 +142,7 @@ class VirtualDrive(object):
         for node_class in all_node_types:
             if prefix == node_class.prefix:
                 node = node_class()
-                node.populate_node(serialized, self.make_node)
+                node.populate_node(serialized, self.make_node_from_serialized)
                 return node
         raise RuntimeError("unable to handle subtree type '%s'" % prefix)