more filetable_new tests

src/allmydata/filetable_new.py

@@ -84,6 +84,16 @@ class ISubTree(Interface):
         either (True, node), or (False, next_subtree_spec, prepath, postpath).
         """
 
+    def serialize():
+        """Return a series of nested lists which describe my structure
+        in a form that can be bencoded."""
+
+    def unserialize(serialized_data):
+        """Populate all nodes from serialized_data, previously created by
+        calling my serialize() method. 'serialized_data' is a series of
+        nested lists (s-expressions), probably recorded in bencoded form."""
+
+
 class IMutableSubTree(Interface):
     def mutation_affects_parent():
         """This returns True for CHK nodes where you must inform the parent
@@ -106,7 +116,7 @@ class IMutableSubTree(Interface):
         everything has been added to the work queue.
         """
 
-    def serialize_to_file():
+    def serialize_to_file(f):
         """Write a bencoded data structure to the given filehandle that can
         be used to reproduce the contents of this subtree."""
 
@@ -191,17 +201,14 @@ class SubTreeNode:
         for i in range(1, len(data), 2):
             name = data[i]
             child_data = data[i+1]
-            assert isinstance(child_data, list)
+            assert isinstance(child_data, (list, tuple))
             child_type = child_data[0]
             if child_type == "DIRECTORY":
                 child = SubTreeNode(self.enclosing_tree)
                 child.unserialize(child_data)
                 self.node_children[name] = child
-            elif child_type == "LINK":
-                self.child_specifications[name] = child_data[1]
             else:
-                raise RuntimeError("unknown serialized-node type '%s'" %
-                                   child_type)
+                self.child_specifications[name] = child_data
 
 class _SubTreeMixin(object):
 

src/allmydata/test/test_filetable_new.py

@@ -2,38 +2,62 @@
 import os
 from zope.interface import implements
 from twisted.trial import unittest
+from twisted.internet import defer
 from allmydata import filetable_new as ft
 from allmydata import workqueue
 from cStringIO import StringIO
 
 class FakeOpener(object):
     implements(ft.IOpener)
+    def __init__(self, objects={}):
+        self.objects = objects
+    def open(self, subtree_specification, parent_is_mutable):
+        #print "open", subtree_specification, subtree_specification.serialize(), parent_is_mutable
+        return defer.succeed(self.objects[subtree_specification.serialize()])
 
 class FakeWorkQueue(object):
     implements(workqueue.IWorkQueue)
+    def __init__(self):
+        self.first_commands = []
+        self.last_commands = []
+        self.tempfile_number = 0
+        self.boxname_number = 0
+    def dump_commands(self):
+        return self.first_commands + self.last_commands
+    def clear_commands(self):
+        self.first_commands = []
+        self.last_commands = []
+
     def create_tempfile(self):
-        return (StringIO(), "dummy_filename")
+        self.tempfile_number += 1
+        self.first_commands.append("create_tempfile-%d" % self.tempfile_number)
+        return (StringIO(), "dummy_filename-%d" % self.tempfile_number)
     def create_boxname(self):
-        return "dummy_boxname"
+        self.boxname_number += 1
+        self.first_commands.append("create_boxname-%d" % self.boxname_number)
+        return "dummy_boxname-%d" % self.boxname_number
     def add_upload_chk(self, source_filename, stash_uri_in_boxname):
-        pass
+        self.first_commands.append(("upload_chk", source_filename,
+                                    stash_uri_in_boxname))
     def add_upload_ssk(self, source_filename, write_capability,
                        previous_version):
-        pass
+        self.first_commands.append(("upload_ssk", source_filename,
+                                    write_capability, previous_version))
     def add_retain_ssk(self, read_capability):
-        pass
+        self.last_commands.append(("retain_ssk", read_capability))
     def add_unlink_ssk(self, write_capability):
-        pass
+        self.last_commands.append(("unlink_ssk", write_capability))
     def add_retain_uri_from_box(self, boxname):
-        pass
+        self.last_commands.append(("retain_uri_from_box", boxname))
     def add_addpath(self, boxname, path):
-        pass
+        self.first_commands.append(("addpath", boxname, path))
     def add_unlink_uri(self, uri):
-        pass
+        self.last_commands.append(("unlink_uri", uri))
     def add_delete_tempfile(self, filename):
-        pass
+        self.first_commands.append(("delete_tempfile", filename))
     def add_delete_box(self, boxname):
-        pass
+        self.last_commands.append(("delete_box", boxname))
+
 
 
 class OneSubTree(unittest.TestCase):
@@ -120,3 +144,158 @@ class OneSubTree(unittest.TestCase):
         d.addCallback(_got_two)
         return d
 
+    def test_addpath(self):
+        o = FakeOpener()
+        wq = FakeWorkQueue()
+        st = ft.MutableCHKDirectorySubTree()
+        st.new()
+        st.set_uri(None)
+        file_three = ft.CHKFileSpecification()
+        file_three.set_uri("file_three_uri")
+        d = st.add(["one", "two", "three"], file_three, o, wq)
+        def _done(res):
+            expected = [
+                "create_tempfile-1",
+                "create_boxname-1",
+                ('upload_chk', 'dummy_filename-1', 'dummy_boxname-1'),
+                ('delete_tempfile', 'dummy_filename-1'),
+                ('addpath', 'dummy_boxname-1', []),
+                ('retain_uri_from_box', 'dummy_boxname-1'),
+                ('delete_box', 'dummy_boxname-1'),
+                ('unlink_uri', None),
+                ]
+            self.failUnlessEqual(wq.dump_commands(), expected)
+            #print
+            #for c in wq.dump_commands():
+            #    print c
+        d.addCallback(_done)
+        return d
+
+    def test_serialize(self):
+        st = ft.ImmutableDirectorySubTree()
+        st.new()
+        one = ft.SubTreeNode(st)
+        two = ft.SubTreeNode(st)
+        three = ft.SubTreeNode(st)
+        st.root.node_children["one"] = one
+        st.root.node_children["two"] = two
+        two.node_children["three"] = three
+        file_four = ft.CHKFileSpecification()
+        file_four.set_uri("file_four_uri")
+        two.child_specifications["four"] = file_four
+        data = st.serialize()
+        st_new = ft.ImmutableDirectorySubTree()
+        st_new.unserialize(data)
+
+        st_four = ft.ImmutableDirectorySubTree()
+        st_four.new()
+        st_four.root.node_children["five"] = ft.SubTreeNode(st_four)
+
+        o = FakeOpener({("CHK-File", "file_four_uri"): st_four})
+        d = st.get([], o)
+        def _got_root(root):
+            self.failUnless(ft.IDirectoryNode.providedBy(root))
+            self.failUnlessEqual(root.list(), ["one", "two"])
+        d.addCallback(_got_root)
+        d.addCallback(lambda res: st.get(["two"], o))
+        def _got_two(_two):
+            self.failUnless(ft.IDirectoryNode.providedBy(_two))
+            self.failUnlessEqual(_two.list(), ["four", "three"])
+        d.addCallback(_got_two)
+
+        d.addCallback(lambda res: st.get(["two", "four"], o))
+        def _got_four(_four):
+            self.failUnless(ft.IDirectoryNode.providedBy(_four))
+            self.failUnlessEqual(_four.list(), ["five"])
+        d.addCallback(_got_four)
+
+class MultipleSubTrees(unittest.TestCase):
+
+    def test_open(self):
+        st = ft.ImmutableDirectorySubTree()
+        st.new()
+        # populate it with some internal directories and child links and see
+        # if we can follow them
+        one = ft.SubTreeNode(st)
+        two = ft.SubTreeNode(st)
+        three = ft.SubTreeNode(st)
+        st.root.node_children["one"] = one
+        st.root.node_children["two"] = two
+        two.node_children["three"] = three
+
+    def test_addpath(self):
+        wq = FakeWorkQueue()
+        st1 = ft.MutableCHKDirectorySubTree()
+        st1.new()
+        st1.set_uri(None)
+        one = ft.SubTreeNode(st1)
+        two = ft.SubTreeNode(st1)
+        st1.root.node_children["one"] = one
+        one.node_children["two"] = two
+        three = ft.CHKDirectorySpecification()
+        three.set_uri("dir_three_uri")
+        two.child_specifications["three"] = three
+
+        st2 = ft.MutableCHKDirectorySubTree()
+        st2.new()
+        st2.set_uri(None)
+        four = ft.SubTreeNode(st2)
+        five = ft.SubTreeNode(st2)
+        st2.root.node_children["four"] = four
+        four.node_children["five"] = five
+
+        file_six = ft.CHKFileSpecification()
+        file_six.set_uri("file_six_uri")
+
+        o = FakeOpener({("CHK-Directory", "dir_three_uri"): st2})
+
+        d = defer.succeed(None)
+        #d.addCallback(lambda res:
+        #              st1.get(["one", "two", "three", "four", "five"], o))
+        def _got_five(res):
+            self.failUnless(ft.IDirectoryNode.providedBy(res))
+            self.failUnlessIdentical(res, five)
+        #d.addCallback(_got_five)
+
+        d.addCallback(lambda res:
+                      st1.add(["one", "two", "six"],
+                              file_six, o, wq))
+        def _done(res):
+            expected = [
+                "create_tempfile-1",
+                "create_boxname-1",
+                ('upload_chk', 'dummy_filename-1', 'dummy_boxname-1'),
+                ('delete_tempfile', 'dummy_filename-1'),
+                # one/two/six only modifies the top-most CHKDirectory, so
+                # the addpath that gets scheduled is targeted at the root
+                ('addpath', 'dummy_boxname-1', []),
+                ('retain_uri_from_box', 'dummy_boxname-1'),
+                ('delete_box', 'dummy_boxname-1'),
+                ('unlink_uri', None),
+                ]
+            self.failUnlessEqual(wq.dump_commands(), expected)
+            wq.clear_commands()
+        d.addCallback(_done)
+
+        d.addCallback(lambda res:
+                      st1.add(["one", "two", "three", "four", "six"],
+                              file_six, o, wq))
+        def _done2(res):
+            expected = [
+                "create_tempfile-2",
+                "create_boxname-2",
+                ('upload_chk', 'dummy_filename-2', 'dummy_boxname-2'),
+                ('delete_tempfile', 'dummy_filename-2'),
+                # one/two/three/four/six modifies the lower CHKDirectory, so
+                # we schedule an addpath of the link that points from the
+                # upper CHKDirectory to the lower one (at one/two/three).
+                ('addpath', 'dummy_boxname-2', ["one", "two", "three"]),
+                ('retain_uri_from_box', 'dummy_boxname-2'),
+                ('delete_box', 'dummy_boxname-2'),
+                ('unlink_uri', None),
+                ]
+            self.failUnlessEqual(wq.dump_commands(), expected)
+        d.addCallback(_done2)
+
+
+        return d