test_dirnode.py: obtain full coverage of dirnode.py

src/allmydata/dirnode.py

@@ -6,8 +6,7 @@ from twisted.internet import defer
 import simplejson
 from allmydata.mutable import NotMutableError
 from allmydata.interfaces import IMutableFileNode, IDirectoryNode,\
-     IURI, IFileNode, \
-     IVerifierURI
+     IURI, IFileNode, IMutableFileURI, IVerifierURI
 from allmydata.util import hashutil
 from allmydata.util.hashutil import netstring
 from allmydata.uri import NewDirectoryURI
@@ -67,7 +66,7 @@ class NewDirectoryNode:
         d.addCallback(self._filenode_created)
         return d
     def _filenode_created(self, res):
-        self._uri = NewDirectoryURI(self._node._uri)
+        self._uri = NewDirectoryURI(IMutableFileURI(self._node.get_uri()))
         return self
 
     def _read(self):
@@ -159,7 +158,7 @@ class NewDirectoryNode:
 
     def check(self):
         """Perform a file check. See IChecker.check for details."""
-        pass # TODO
+        return defer.succeed(None) # TODO
 
     def list(self):
         """I return a Deferred that fires with a dictionary mapping child

src/allmydata/interfaces.py

@@ -1135,6 +1135,33 @@ class IChecker(Interface):
         might need to back away from this in the future.
         """
 
+class IClient(Interface):
+    def upload(uploadable, wait_for_numpeers):
+        """Upload some data into a CHK, get back the URI string for it.
+        @param uploadable: something that implements IUploadable
+        @param wait_for_numpeers: don't upload anything until we have at least
+                                  this many peers connected
+        @return: a Deferred that fires with the (string) URI for this file.
+        """
+    def create_empty_dirnode(wait_for_numpeers):
+        """Create a new dirnode, empty and unattached.
+        @param wait_for_numpeers: don't create anything until we have at least
+                                  this many peers connected.
+        @return: a Deferred that fires with the new IDirectoryNode instance.
+        """
+    def create_node_from_uri(uri):
+        """Create a new IFilesystemNode instance from the uri, synchronously.
+        @param uri: a string or IURI-providing instance. This could be for a
+                    LiteralFileNode, a CHK file node, a mutable file node, or
+                    a directory node
+        @return: an instance that provides IFilesystemNode (or more usefully one
+                 of its subclasses). File-specifying URIs will result in
+                 IFileNode or IMutableFileNode -providing instances, like
+                 FileNode, LiteralFileNode, or MutableFileNode.
+                 Directory-specifying URIs will result in
+                 IDirectoryNode-providing instances, like NewDirectoryNode.
+        """
+
 
 class NotCapableError(Exception):
     """You have tried to write to a read-only node."""

src/allmydata/test/test_dirnode.py

@@ -0,0 +1,337 @@
+
+import os
+from zope.interface import implements
+from twisted.trial import unittest
+from twisted.internet import defer
+from allmydata import uri, dirnode, upload
+from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
+     INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode
+from allmydata.util import hashutil, testutil
+
+# to test dirnode.py, we want to construct a tree of real DirectoryNodes that
+# contain pointers to fake files. We start with a fake MutableFileNode that
+# stores all of its data in a static table.
+
+def make_chk_file_uri(size):
+    return uri.CHKFileURI(key=os.urandom(16),
+                          uri_extension_hash=os.urandom(32),
+                          needed_shares=3,
+                          total_shares=10,
+                          size=size)
+
+def make_mutable_file_uri():
+    return uri.WriteableSSKFileURI(writekey=os.urandom(16),
+                                   fingerprint=os.urandom(32))
+def make_verifier_uri():
+    return uri.SSKVerifierURI(storage_index=os.urandom(16),
+                              fingerprint=os.urandom(32))
+
+class FakeMutableFileNode:
+    implements(IMutableFileNode)
+    all_contents = {}
+    def __init__(self, client):
+        self.client = client
+        self.my_uri = make_mutable_file_uri()
+        self.storage_index = self.my_uri.storage_index
+    def create(self, initial_contents, wait_for_numpeers=None):
+        self.all_contents[self.storage_index] = initial_contents
+        return defer.succeed(self)
+    def init_from_uri(self, myuri):
+        self.my_uri = IURI(myuri)
+        self.storage_index = self.my_uri.storage_index
+        return self
+    def get_uri(self):
+        return self.my_uri
+    def is_readonly(self):
+        return self.my_uri.is_readonly()
+    def is_mutable(self):
+        return self.my_uri.is_mutable()
+    def download_to_data(self):
+        return defer.succeed(self.all_contents[self.storage_index])
+    def get_writekey(self):
+        return "\x00"*16
+
+    def replace(self, new_contents, wait_for_numpeers=None):
+        self.all_contents[self.storage_index] = new_contents
+        return defer.succeed(None)
+
+class MyDirectoryNode(dirnode.NewDirectoryNode):
+    filenode_class = FakeMutableFileNode
+
+class Marker:
+    implements(IFileNode, IMutableFileNode) # sure, why not
+    def __init__(self, nodeuri):
+        if not isinstance(nodeuri, str):
+            nodeuri = nodeuri.to_string()
+        self.nodeuri = nodeuri
+        si = hashutil.tagged_hash("tag1", nodeuri)
+        fp = hashutil.tagged_hash("tag2", nodeuri)
+        self.verifieruri = uri.SSKVerifierURI(storage_index=si,
+                                              fingerprint=fp).to_string()
+    def get_uri(self):
+        return self.nodeuri
+    def get_readonly_uri(self):
+        return self.nodeuri
+    def get_verifier(self):
+        return self.verifieruri
+
+# dirnode requires three methods from the client: upload(),
+# create_node_from_uri(), and create_empty_dirnode(). Of these, upload() is
+# only used by the convenience composite method add_file().
+
+class FakeClient:
+    implements(IClient)
+    chk_contents = {}
+
+    def upload(self, uploadable, wait_for_numpeers):
+        d = uploadable.get_size()
+        d.addCallback(lambda size: uploadable.read(size))
+        def _got_data(datav):
+            data = "".join(datav)
+            u = make_chk_file_uri(len(data))
+            self.chk_contents[u] = data
+            return u
+        d.addCallback(_got_data)
+        return d
+
+    def create_node_from_uri(self, u):
+        u = IURI(u)
+        if (INewDirectoryURI.providedBy(u)
+            or IReadonlyNewDirectoryURI.providedBy(u)):
+            return MyDirectoryNode(self).init_from_uri(u)
+        return Marker(u.to_string())
+
+    def create_empty_dirnode(self, wait_for_numpeers):
+        n = MyDirectoryNode(self)
+        d = n.create(wait_for_numpeers)
+        d.addCallback(lambda res: n)
+        return d
+
+
+class Dirnode(unittest.TestCase, testutil.ShouldFailMixin):
+    def setUp(self):
+        self.client = FakeClient()
+
+    def test_basic(self):
+        d = self.client.create_empty_dirnode(0)
+        def _done(res):
+            self.failUnless(isinstance(res, MyDirectoryNode))
+            rep = str(res)
+            self.failUnless("RW" in rep)
+        d.addCallback(_done)
+        return d
+
+    def test_corrupt(self):
+        d = self.client.create_empty_dirnode(0)
+        def _created(dn):
+            u = make_mutable_file_uri()
+            d = dn.set_uri("child", u)
+            d.addCallback(lambda res: dn.list())
+            def _check1(children):
+                self.failUnless("child" in children)
+            d.addCallback(_check1)
+            d.addCallback(lambda res:
+                          self.shouldFail(KeyError, "get bogus", None,
+                                          dn.get, "bogus"))
+            def _corrupt(res):
+                filenode = dn._node
+                si = IURI(filenode.get_uri()).storage_index
+                old_contents = filenode.all_contents[si]
+                # we happen to know that the writecap is encrypted near the
+                # end of the string. Flip one of its bits and make sure we
+                # detect the corruption.
+                new_contents = testutil.flip_bit(old_contents, -10)
+                filenode.all_contents[si] = new_contents
+            d.addCallback(_corrupt)
+            def _check2(res):
+                self.shouldFail(hashutil.IntegrityCheckError, "corrupt",
+                                "HMAC does not match, crypttext is corrupted",
+                                dn.list)
+            d.addCallback(_check2)
+            return d
+        d.addCallback(_created)
+        return d
+
+    def test_check(self):
+        d = self.client.create_empty_dirnode(0)
+        d.addCallback(lambda dn: dn.check())
+        def _done(res):
+            pass
+        d.addCallback(_done)
+        return d
+
+    def test_readonly(self):
+        fileuri = make_chk_file_uri(1234)
+        filenode = self.client.create_node_from_uri(fileuri)
+        uploadable = upload.Data("some data")
+
+        d = self.client.create_empty_dirnode(0)
+        def _created(rw_dn):
+            d2 = rw_dn.set_uri("child", fileuri)
+            d2.addCallback(lambda res: rw_dn)
+            return d2
+        d.addCallback(_created)
+
+        def _ready(rw_dn):
+            ro_uri = rw_dn.get_readonly_uri()
+            ro_dn = self.client.create_node_from_uri(ro_uri)
+            self.failUnless(ro_dn.is_readonly())
+            self.failUnless(ro_dn.is_mutable())
+
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.set_uri, "newchild", fileuri)
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.set_node, "newchild", filenode)
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.add_file, "newchild", uploadable)
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.delete, "child")
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.create_empty_directory, "newchild")
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            ro_dn.move_child_to, "child", rw_dn)
+            self.shouldFail(dirnode.NotMutableError, "set_uri ro", None,
+                            rw_dn.move_child_to, "child", ro_dn)
+            return ro_dn.list()
+        d.addCallback(_ready)
+        def _listed(children):
+            self.failUnless("child" in children)
+        d.addCallback(_listed)
+        return d
+
+    def test_create(self):
+        self.expected_manifest = []
+
+        d = self.client.create_empty_dirnode(wait_for_numpeers=1)
+        def _then(n):
+            self.failUnless(n.is_mutable())
+            u = n.get_uri()
+            self.failUnless(u)
+            self.failUnless(u.startswith("URI:DIR2:"), u)
+            u_ro = n.get_readonly_uri()
+            self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
+            u_v = n.get_verifier()
+            self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
+            self.expected_manifest.append(u_v)
+
+            d = n.list()
+            d.addCallback(lambda res: self.failUnlessEqual(res, {}))
+            d.addCallback(lambda res: n.has_child("missing"))
+            d.addCallback(lambda res: self.failIf(res))
+            fake_file_uri = make_mutable_file_uri()
+            m = Marker(fake_file_uri)
+            ffu_v = m.get_verifier()
+            assert isinstance(ffu_v, str)
+            self.expected_manifest.append(ffu_v)
+            d.addCallback(lambda res: n.set_uri("child", fake_file_uri))
+
+            d.addCallback(lambda res: n.create_empty_directory("subdir", wait_for_numpeers=1))
+            def _created(subdir):
+                self.failUnless(isinstance(subdir, MyDirectoryNode))
+                self.subdir = subdir
+                new_v = subdir.get_verifier()
+                assert isinstance(new_v, str)
+                self.expected_manifest.append(new_v)
+            d.addCallback(_created)
+
+            d.addCallback(lambda res: n.list())
+            d.addCallback(lambda children:
+                          self.failUnlessEqual(sorted(children.keys()),
+                                               sorted(["child", "subdir"])))
+
+            d.addCallback(lambda res: n.build_manifest())
+            def _check_manifest(manifest):
+                self.failUnlessEqual(sorted(manifest),
+                                     sorted(self.expected_manifest))
+            d.addCallback(_check_manifest)
+
+            def _add_subsubdir(res):
+                return self.subdir.create_empty_directory("subsubdir", wait_for_numpeers=1)
+            d.addCallback(_add_subsubdir)
+            d.addCallback(lambda res: n.get_child_at_path("subdir/subsubdir"))
+            d.addCallback(lambda subsubdir:
+                          self.failUnless(isinstance(subsubdir,
+                                                     MyDirectoryNode)))
+            d.addCallback(lambda res: n.get_child_at_path(""))
+            d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
+                                                           n.get_uri()))
+
+            d.addCallback(lambda res: n.get_metadata_for("child"))
+            d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
+
+            d.addCallback(lambda res: n.delete("subdir"))
+            d.addCallback(lambda old_child:
+                          self.failUnlessEqual(old_child.get_uri(),
+                                               self.subdir.get_uri()))
+
+            d.addCallback(lambda res: n.list())
+            d.addCallback(lambda children:
+                          self.failUnlessEqual(sorted(children.keys()),
+                                               sorted(["child"])))
+
+            uploadable = upload.Data("some data")
+            d.addCallback(lambda res: n.add_file("newfile", uploadable))
+            d.addCallback(lambda newnode:
+                          self.failUnless(IFileNode.providedBy(newnode)))
+            d.addCallback(lambda res: n.list())
+            d.addCallback(lambda children:
+                          self.failUnlessEqual(sorted(children.keys()),
+                                               sorted(["child", "newfile"])))
+
+            d.addCallback(lambda res: n.create_empty_directory("subdir2"))
+            def _created2(subdir2):
+                self.subdir2 = subdir2
+            d.addCallback(_created2)
+
+            d.addCallback(lambda res:
+                          n.move_child_to("child", self.subdir2))
+            d.addCallback(lambda res: n.list())
+            d.addCallback(lambda children:
+                          self.failUnlessEqual(sorted(children.keys()),
+                                               sorted(["newfile", "subdir2"])))
+            d.addCallback(lambda res: self.subdir2.list())
+            d.addCallback(lambda children:
+                          self.failUnlessEqual(sorted(children.keys()),
+                                               sorted(["child"])))
+
+            return d
+
+        d.addCallback(_then)
+
+        return d
+
+netstring = hashutil.netstring
+split_netstring = dirnode.split_netstring
+
+class Netstring(unittest.TestCase):
+    def test_split(self):
+        a = netstring("hello") + netstring("world")
+        self.failUnlessEqual(split_netstring(a, 2), ("hello", "world"))
+        self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world"))
+        self.failUnlessEqual(split_netstring(a, 2, True),
+                             ("hello", "world", ""))
+        self.failUnlessRaises(ValueError, split_netstring, a, 3)
+        self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2)
+        self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False)
+
+    def test_extra(self):
+        a = netstring("hello")
+        self.failUnlessEqual(split_netstring(a, 1, True), ("hello", ""))
+        b = netstring("hello") + "extra stuff"
+        self.failUnlessEqual(split_netstring(b, 1, True),
+                             ("hello", "extra stuff"))
+
+    def test_nested(self):
+        a = netstring("hello") + netstring("world") + "extra stuff"
+        b = netstring("a") + netstring("is") + netstring(a) + netstring(".")
+        top = split_netstring(b, 4)
+        self.failUnlessEqual(len(top), 4)
+        self.failUnlessEqual(top[0], "a")
+        self.failUnlessEqual(top[1], "is")
+        self.failUnlessEqual(top[2], a)
+        self.failUnlessEqual(top[3], ".")
+        self.failUnlessRaises(ValueError, split_netstring, a, 2)
+        self.failUnlessRaises(ValueError, split_netstring, a, 2, False)
+        bottom = split_netstring(a, 2, True)
+        self.failUnlessEqual(bottom, ("hello", "world", "extra stuff"))
+

src/allmydata/test/test_mutable.py

@@ -3,46 +3,14 @@ import itertools, struct
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.python import failure, log
-from allmydata import mutable, uri, dirnode, upload
-from allmydata.dirnode import split_netstring
-from allmydata.util.hashutil import netstring, tagged_hash
+from allmydata import mutable, uri, dirnode
+from allmydata.util.hashutil import tagged_hash
 from allmydata.encode import NotEnoughPeersError
 from allmydata.interfaces import IURI, INewDirectoryURI, \
-     IMutableFileURI, IFileNode, IUploadable, IFileURI
+     IMutableFileURI, IUploadable, IFileURI
 from allmydata.filenode import LiteralFileNode
 import sha
 
-class Netstring(unittest.TestCase):
-    def test_split(self):
-        a = netstring("hello") + netstring("world")
-        self.failUnlessEqual(split_netstring(a, 2), ("hello", "world"))
-        self.failUnlessEqual(split_netstring(a, 2, False), ("hello", "world"))
-        self.failUnlessEqual(split_netstring(a, 2, True),
-                             ("hello", "world", ""))
-        self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2)
-        self.failUnlessRaises(ValueError, split_netstring, a+" extra", 2, False)
-
-    def test_extra(self):
-        a = netstring("hello")
-        self.failUnlessEqual(split_netstring(a, 1, True), ("hello", ""))
-        b = netstring("hello") + "extra stuff"
-        self.failUnlessEqual(split_netstring(b, 1, True),
-                             ("hello", "extra stuff"))
-
-    def test_nested(self):
-        a = netstring("hello") + netstring("world") + "extra stuff"
-        b = netstring("a") + netstring("is") + netstring(a) + netstring(".")
-        top = split_netstring(b, 4)
-        self.failUnlessEqual(len(top), 4)
-        self.failUnlessEqual(top[0], "a")
-        self.failUnlessEqual(top[1], "is")
-        self.failUnlessEqual(top[2], a)
-        self.failUnlessEqual(top[3], ".")
-        self.failUnlessRaises(ValueError, split_netstring, a, 2)
-        self.failUnlessRaises(ValueError, split_netstring, a, 2, False)
-        bottom = split_netstring(a, 2, True)
-        self.failUnlessEqual(bottom, ("hello", "world", "extra stuff"))
-
 class FakeFilenode(mutable.MutableFileNode):
     counter = itertools.count(1)
     all_contents = {}
@@ -441,106 +409,3 @@ class FakePrivKey:
         return "PRIVKEY-%d" % self.count
     def sign(self, data):
         return "SIGN(%s)" % data
-
-class Dirnode(unittest.TestCase):
-    def setUp(self):
-        self.client = FakeClient()
-
-    def test_create(self):
-        self.expected_manifest = []
-
-        d = self.client.create_empty_dirnode(wait_for_numpeers=1)
-        def _then(n):
-            self.failUnless(n.is_mutable())
-            u = n.get_uri()
-            self.failUnless(u)
-            self.failUnless(u.startswith("URI:DIR2:"), u)
-            u_ro = n.get_readonly_uri()
-            self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
-            u_v = n.get_verifier()
-            self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
-            self.expected_manifest.append(u_v)
-
-            d = n.list()
-            d.addCallback(lambda res: self.failUnlessEqual(res, {}))
-            d.addCallback(lambda res: n.has_child("missing"))
-            d.addCallback(lambda res: self.failIf(res))
-            fake_file_uri = uri.WriteableSSKFileURI("a"*16,"b"*32)
-            ffu_v = fake_file_uri.get_verifier().to_string()
-            self.expected_manifest.append(ffu_v)
-            d.addCallback(lambda res: n.set_uri("child", fake_file_uri))
-
-            d.addCallback(lambda res: n.create_empty_directory("subdir", wait_for_numpeers=1))
-            def _created(subdir):
-                self.failUnless(isinstance(subdir, FakeNewDirectoryNode))
-                self.subdir = subdir
-                new_v = subdir.get_verifier()
-                self.expected_manifest.append(new_v)
-            d.addCallback(_created)
-
-            d.addCallback(lambda res: n.list())
-            d.addCallback(lambda children:
-                          self.failUnlessEqual(sorted(children.keys()),
-                                               sorted(["child", "subdir"])))
-
-            d.addCallback(lambda res: n.build_manifest())
-            def _check_manifest(manifest):
-                self.failUnlessEqual(sorted(manifest),
-                                     sorted(self.expected_manifest))
-            d.addCallback(_check_manifest)
-
-            def _add_subsubdir(res):
-                return self.subdir.create_empty_directory("subsubdir", wait_for_numpeers=1)
-            d.addCallback(_add_subsubdir)
-            d.addCallback(lambda res: n.get_child_at_path("subdir/subsubdir"))
-            d.addCallback(lambda subsubdir:
-                          self.failUnless(isinstance(subsubdir,
-                                                     FakeNewDirectoryNode)))
-            d.addCallback(lambda res: n.get_child_at_path(""))
-            d.addCallback(lambda res: self.failUnlessEqual(res.get_uri(),
-                                                           n.get_uri()))
-
-            d.addCallback(lambda res: n.get_metadata_for("child"))
-            d.addCallback(lambda metadata: self.failUnlessEqual(metadata, {}))
-
-            d.addCallback(lambda res: n.delete("subdir"))
-            d.addCallback(lambda old_child:
-                          self.failUnlessEqual(old_child.get_uri(),
-                                               self.subdir.get_uri()))
-
-            d.addCallback(lambda res: n.list())
-            d.addCallback(lambda children:
-                          self.failUnlessEqual(sorted(children.keys()),
-                                               sorted(["child"])))
-
-            uploadable = upload.Data("some data")
-            d.addCallback(lambda res: n.add_file("newfile", uploadable))
-            d.addCallback(lambda newnode:
-                          self.failUnless(IFileNode.providedBy(newnode)))
-            d.addCallback(lambda res: n.list())
-            d.addCallback(lambda children:
-                          self.failUnlessEqual(sorted(children.keys()),
-                                               sorted(["child", "newfile"])))
-
-            d.addCallback(lambda res: n.create_empty_directory("subdir2"))
-            def _created2(subdir2):
-                self.subdir2 = subdir2
-            d.addCallback(_created2)
-
-            d.addCallback(lambda res:
-                          n.move_child_to("child", self.subdir2))
-            d.addCallback(lambda res: n.list())
-            d.addCallback(lambda children:
-                          self.failUnlessEqual(sorted(children.keys()),
-                                               sorted(["newfile", "subdir2"])))
-            d.addCallback(lambda res: self.subdir2.list())
-            d.addCallback(lambda children:
-                          self.failUnlessEqual(sorted(children.keys()),
-                                               sorted(["child"])))
-
-            return d
-
-        d.addCallback(_then)
-
-        return d
-

src/allmydata/util/testutil.py

@@ -1,6 +1,16 @@
 import os, signal, time
 
 from twisted.internet import reactor, defer
+from twisted.python import failure
+
+
+def flip_bit(good, which):
+    # flip the low-order bit of good[which]
+    if which == -1:
+        pieces = good[:which], good[-1:], ""
+    else:
+        pieces = good[:which], good[which:which+1], good[which+1:]
+    return pieces[0] + chr(ord(pieces[1]) ^ 0x01) + pieces[2]
 
 class SignalMixin:
     # This class is necessary for any code which wants to use Processes
@@ -37,6 +47,24 @@ class PollMixin:
         reactor.callLater(pollinterval, d.callback, None)
         return d
 
+class ShouldFailMixin:
+
+    def shouldFail(self, expected_failure, which, substring, callable, *args, **kwargs):
+        assert substring is None or isinstance(substring, str)
+        d = defer.maybeDeferred(callable, *args, **kwargs)
+        def done(res):
+            if isinstance(res, failure.Failure):
+                res.trap(expected_failure)
+                if substring:
+                    self.failUnless(substring in str(res),
+                                    "substring '%s' not in '%s'"
+                                    % (substring, str(res)))
+            else:
+                self.fail("%s was supposed to raise %s, not get '%s'" %
+                          (which, expected_failure, res))
+        d.addBoth(done)
+        return d
+
 
 class TestMixin(SignalMixin):
     def setUp(self, repeatable=False):