more filetree work, more tests now pass

src/allmydata/filetree/directory.py

@@ -3,7 +3,7 @@ from zope.interface import implements
 from twisted.internet import defer
 from cStringIO import StringIO
 from allmydata.filetree.interfaces import (
-    INode, IDirectoryNode, ISubTree,
+    INode, INodeMaker, IDirectoryNode, ISubTree,
     ICHKDirectoryNode, ISSKDirectoryNode,
     NoSuchChildError,
     )
@@ -19,17 +19,14 @@ from allmydata.util import bencode
 #  each time the vdrive changes, update the local drive to match, and
 #  vice versa.
 
-# from the itertools 'recipes' page
+from itertools import islice, izip
-from itertools import izip, tee
+def in_pairs(iterable):
-def pairwise(iterable):
+    "s -> (s0,s1), (s2,s3), (s4,s5), ..."
-    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
+    a = islice(iterable, 0, None, 2)
-    a, b = tee(iterable)
+    b = islice(iterable, 1, None, 2)
-    try:
-        b.next()
-    except StopIteration:
-        pass
     return izip(a, b)
 
+
 class SubTreeNode:
     implements(INode, IDirectoryNode)
 
@@ -95,15 +92,16 @@ class SubTreeNode:
             data.append(self.children[name].serialize_node())
         return data
 
-    def populate_node(self, data, node_maker):
+    def populate_dirnode(self, data, node_maker):
+        assert INodeMaker(node_maker)
         assert len(data) % 2 == 0
-        for (name, child_data) in pairwise(data):
+        for (name, child_data) in in_pairs(data):
             if isinstance(child_data, (list, tuple)):
                 child = SubTreeNode(self.enclosing_tree)
-                child.populate_node(child_data)
+                child.populate_dirnode(child_data, node_maker)
             else:
                 assert isinstance(child_data, str)
-                child = node_maker(child_data)
+                child = node_maker.make_node_from_serialized(child_data)
             self.children[name] = child
 
 
@@ -141,7 +139,7 @@ class _DirectorySubTree(object):
 
     def _populate_from_data(self, data, node_maker):
         self.root = SubTreeNode(self)
-        self.root.populate_node(bencode.bdecode(data), node_maker)
+        self.root.populate_dirnode(bencode.bdecode(data), node_maker)
         return self
 
     def serialize_subtree_to_file(self, f):
@@ -205,14 +203,16 @@ class LocalFileSubTree(_DirectorySubTree):
         f = open(self.filename, "rb")
         data = f.read()
         f.close()
-        return defer.succeed(self._populate_from_data(node_maker))
+        d = defer.succeed(data)
+        d.addCallback(self._populate_from_data, node_maker)
+        return d
 
     def create_node_now(self):
         return LocalFileSubTreeNode().new(self.filename)
 
     def _update(self):
         f = open(self.filename, "wb")
-        self.serialize_to_file(f)
+        self.serialize_subtree_to_file(f)
         f.close()
 
     def update_now(self, uploader):
@@ -258,7 +258,7 @@ class CHKDirectorySubTree(_DirectorySubTree):
 
     def update_now(self, uploader):
         f = StringIO()
-        self.serialize_to_file(f)
+        self.serialize_subtree_to_file(f)
         data = f.getvalue()
         d = uploader.upload_data(data)
         def _uploaded(uri):
@@ -271,7 +271,7 @@ class CHKDirectorySubTree(_DirectorySubTree):
         # this is the CHK form
         old_uri = self.uri
         f, filename = workqueue.create_tempfile(".chkdir")
-        self.serialize_to_file(f)
+        self.serialize_subtree_to_file(f)
         f.close()
         boxname = workqueue.create_boxname()
         workqueue.add_upload_chk(filename, boxname)
@@ -339,7 +339,7 @@ class SSKDirectorySubTree(_DirectorySubTree):
             raise RuntimeError("This SSKDirectorySubTree is not mutable")
 
         f = StringIO()
-        self.serialize_to_file(f)
+        self.serialize_subtree_to_file(f)
         data = f.getvalue()
 
         self.version += 1
@@ -350,7 +350,7 @@ class SSKDirectorySubTree(_DirectorySubTree):
     def update(self, workqueue):
         # this is the SSK form
         f, filename = workqueue.create_tempfile(".sskdir")
-        self.serialize_to_file(f)
+        self.serialize_subtree_to_file(f)
         f.close()
 
         oldversion = self.version

src/allmydata/filetree/interfaces.py

@@ -17,8 +17,11 @@ class INode(Interface):
         .prefix ."""
     def populate_node(body, node_maker):
         """vdrive.make_node_from_serialized() will first use the prefix from
-        the .prefix attribute to decide what kind of Node to create. It will
+        the .prefix attribute to decide what kind of Node to create. They
-        then call this function with the body to populate the new Node."""
+        will then call this populate_node() method with the body to populate
+        the new Node. 'node_maker' provides INodeMaker, which provides that
+        same make_node_from_serialized function to create any internal child
+        nodes that might be necessary."""
 
 class IFileNode(Interface):
     """This is a file which can be retrieved."""
@@ -190,6 +193,21 @@ class ISubTree(Interface):
         (SSKDirectorySubTrees and redirections).
         """
 
+class INodeMaker(Interface):
+    def make_node_from_serialized(serialized):
+        """Turn a string into an INode, which contains information about
+        the file or directory (like a URI), but does not contain the actual
+        contents. An IOpener can be used later to retrieve the contents
+        (which means downloading the file if this is an IFileNode, or
+        perhaps creating a new subtree from the contents)."""
+
+class ISubTreeMaker(Interface):
+    def make_subtree_from_node(node, parent_is_mutable):
+        """Turn an INode into an ISubTree (using an internal opener to
+        download the data, if necessary).
+        This returns a Deferred that fires with the ISubTree instance.
+        """
+
 #class IMutableSubTree(Interface):
 #    def mutation_affects_parent():
 #        """This returns True for CHK nodes where you must inform the parent

src/allmydata/filetree/opener.py

@@ -3,7 +3,7 @@ from zope.interface import implements
 from twisted.internet import defer
 from allmydata.filetree import interfaces, directory, redirect
 #from allmydata.filetree.file import CHKFile, MutableSSKFile, ImmutableSSKFile
-from allmydata.filetree.interfaces import INode, IDirectoryNode
+from allmydata.filetree.interfaces import INode, IDirectoryNode, INodeMaker
 
 all_openable_subtree_types = [
     directory.LocalFileSubTree,
@@ -27,6 +27,7 @@ class Opener(object):
 
     def _create(self, node, parent_is_mutable, node_maker):
         assert INode(node)
+        assert INodeMaker(node_maker)
         for subtree_class in all_openable_subtree_types:
             if isinstance(node, subtree_class.node_class):
                 subtree = subtree_class()
@@ -41,6 +42,7 @@ class Opener(object):
     def open(self, node, parent_is_mutable, node_maker):
         assert INode(node)
         assert not isinstance(node, IDirectoryNode)
+        assert INodeMaker(node_maker)
 
         # is it in cache? To check this we need to use the node's serialized
         # form, since nodes are instances and don't compare by value

src/allmydata/filetree/redirect.py

@@ -3,7 +3,7 @@ from cStringIO import StringIO
 from zope.interface import implements
 from twisted.internet import defer
 
-from allmydata.filetree.interfaces import ISubTree
+from allmydata.filetree.interfaces import ISubTree, INodeMaker
 from allmydata.filetree.basenode import BaseDataNode
 from allmydata.util import bencode
 
@@ -33,7 +33,8 @@ class _BaseRedirection(object):
         return self.child_node.serialize_node()
 
     def _populate_from_data(self, data, node_maker):
-        self.child_node = node_maker(data)
+        assert INodeMaker(node_maker)
+        self.child_node = node_maker.make_node_from_serialized(data)
         return self
 
 
@@ -64,7 +65,9 @@ class LocalFileRedirection(_BaseRedirection):
         # note: we don't cache the contents of the file. TODO: consider
         # doing this based upon mtime. It is important that we be able to
         # notice if the file has been changed.
-        return defer.succeed(self._populate_from_data(data, node_maker))
+        d = defer.succeed(data)
+        d.addCallback(self._populate_from_data, node_maker)
+        return d
 
     def is_mutable(self):
         return True
@@ -161,10 +164,11 @@ class QueenOrLocalFileRedirection(_BaseRedirection):
         # TODO: queen?
         # TODO: pubsub so we can cache the queen's results
         d = self._queen.callRemote("lookup_handle", self.handle)
-        d.addCallback(self._choose_winner, local_version_and_data, node_maker)
+        d.addCallback(self._choose_winner, local_version_and_data)
+        d.addCallback(self._populate_from_data, node_maker)
         return d
 
-    def _choose_winner(self, queen_version_and_data, local_version_and_data, node_maker):
+    def _choose_winner(self, queen_version_and_data, local_version_and_data):
         queen_version, queen_data = bencode.bdecode(queen_version_and_data)
         local_version, local_data = bencode.bdecode(local_version_and_data)
         if queen_version > local_version:
@@ -174,7 +178,7 @@ class QueenOrLocalFileRedirection(_BaseRedirection):
             data = local_data
             self.version = local_version
         # NOTE: two layers of bencoding here, TODO
-        return self._populate_from_data(data, node_maker)
+        return data
 
     def is_mutable(self):
         return True

src/allmydata/filetree/vdrive.py

@@ -1,8 +1,8 @@
 
 from zope.interface import implements
-from allmydata.filetree import opener, directory, redirect
+from allmydata.filetree import opener, directory, file, redirect
 from allmydata.filetree.interfaces import (
-    IVirtualDrive, INode, ISubTree, IFileNode, IDirectoryNode,
+    IVirtualDrive, INodeMaker, INode, ISubTree, IFileNode, IDirectoryNode,
     NoSuchDirectoryError, NoSuchChildError, PathAlreadyExistsError,
     PathDoesNotExistError,
     )
@@ -12,8 +12,11 @@ from allmydata.upload import IUploadable
 # node specification strings (found inside the serialized form of subtrees)
 # into Nodes (which live in the in-RAM form of subtrees).
 all_node_types = [
+    directory.LocalFileSubTreeNode,
     directory.CHKDirectorySubTreeNode,
     directory.SSKDirectorySubTreeNode,
+    file.CHKFileNode,
+    file.SSKFileNode,
     redirect.LocalFileRedirectionNode,
     redirect.QueenRedirectionNode,
     redirect.HTTPRedirectionNode,
@@ -21,7 +24,7 @@ all_node_types = [
 ]
 
 class VirtualDrive(object):
-    implements(IVirtualDrive)
+    implements(IVirtualDrive, INodeMaker)
 
     def __init__(self, workqueue, downloader, root_node):
         assert INode(root_node)
@@ -33,6 +36,8 @@ class VirtualDrive(object):
         self.root_node = root_node
 
     # these are called when loading and creating nodes
+
+    # INodeMaker
     def make_node_from_serialized(self, serialized):
         # this turns a string into an INode, which contains information about
         # the file or directory (like a URI), but does not contain the actual
@@ -47,14 +52,14 @@ class VirtualDrive(object):
         for node_class in all_node_types:
             if prefix == node_class.prefix:
                 node = node_class()
-                node.populate_node(body, self.make_node_from_serialized)
+                node.populate_node(body, self)
                 return node
-        raise RuntimeError("unable to handle subtree type '%s'" % prefix)
+        raise RuntimeError("unable to handle node type '%s'" % prefix)
 
+    # ISubTreeMaker
     def make_subtree_from_node(self, node, parent_is_mutable):
         assert INode(node)
-        return self.opener.open(node, parent_is_mutable,
+        return self.opener.open(node, parent_is_mutable, self)
-                                self.make_subtree_from_node)
 
     # these methods are used to walk through our subtrees
 

src/allmydata/test/test_filetree_new.py

@@ -1,7 +1,7 @@
 
 #from zope.interface import implements
 from twisted.trial import unittest
-#from twisted.internet import defer
+from twisted.internet import defer
 #from allmydata.filetree.interfaces import IOpener, IDirectoryNode
 #from allmydata.filetree.directory import (ImmutableDirectorySubTree,
 #                                          SubTreeNode,
@@ -314,8 +314,15 @@ class Redirect(unittest.TestCase):
 
 import os.path
 from allmydata.filetree import directory, redirect, vdrive
-from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode)
+from allmydata.filetree.interfaces import (ISubTree, INode, IDirectoryNode, IFileNode)
 from allmydata.filetree.file import CHKFileNode
+from allmydata.util import bencode
+
+class InPairs(unittest.TestCase):
+    def test_in_pairs(self):
+        l = range(8)
+        pairs = list(directory.in_pairs(l))
+        self.failUnlessEqual(pairs, [(0,1), (2,3), (4,5), (6,7)])
 
 class Stuff(unittest.TestCase):
 
@@ -372,11 +379,14 @@ class Stuff(unittest.TestCase):
         del root, subdir1, subdir2, subdir3, subdir4
         # leaving file1 for later use
 
-        # now serialize it and reconstruct it
+        # now serialize it and examine the results
         f = StringIO()
         subtree.serialize_subtree_to_file(f)
         data = f.getvalue()
         #print data
+        unpacked = bencode.bdecode(data)
+        #print unpacked
+        del f, data, unpacked
 
         node = subtree.create_node_now()
         self.failUnless(isinstance(node, directory.LocalFileSubTreeNode))
@@ -384,9 +394,14 @@ class Stuff(unittest.TestCase):
         self.failUnless(isinstance(node_s, str))
         self.failUnless(node_s.startswith("LocalFileDirectory:"))
         self.failUnless("dirtree.save" in node_s)
+        del node, node_s
+
+        d = defer.maybeDeferred(subtree.update_now, None)
+        def _updated(node):
+            # now reconstruct it
+            return v.make_subtree_from_node(node, False)
+        d.addCallback(_updated)
 
-        # now reconstruct it
-        d = v.make_subtree_from_node(node, False)
         def _opened(new_subtree):
             res = new_subtree.get_node_for_path([])
             (found_path, root, remaining_path) = res
@@ -396,7 +411,9 @@ class Stuff(unittest.TestCase):
             self.failUnless(IDirectoryNode.providedBy(root))
             self.failUnlessEqual(root.list(), ["foo.txt", "subdir1"])
             file1a = root.get("foo.txt")
-            self.failUnless(isinstance(CHKFileNode, file1a))
+            self.failUnless(INode(file1a))
+            self.failUnless(isinstance(file1a, CHKFileNode))
+            self.failUnless(IFileNode(file1a))
             self.failUnlessEqual(file1a.get_uri(), "uri1")
             subdir1 = root.get("subdir1")
             subdir2 = subdir1.get("subdir2")
@@ -405,7 +422,6 @@ class Stuff(unittest.TestCase):
             self.failUnlessEqual(subdir2.list(), [])
         d.addCallback(_opened)
         return d
-    testDirectory.todo = "not working yet"
 
     def testVdrive(self):
         # create some stuff, see if we can import everything