web: change t=manifest to return a list of (path,read/writecap) tuples, instead of a list of verifycaps. Add output=html,text,json.

docs/webapi.txt

@@ -871,7 +871,17 @@ POST $URL?t=deep-check&repair=true
 GET $DIRURL?t=manifest
 
   Return an HTML-formatted manifest of the given directory, for debugging.
-  This is a table of verifier-caps.
+  This is a table of (path, filecap/dircap), for every object reachable from
+  the starting directory. The path will be slash-joined, and the
+  filecap/dircap will contain a link to the object in question. This page
+  gives immediate access to every object in the virtual filesystem subtree.
+
+  If output=text is added to the query args, the results will be a text/plain
+  list, with one file/dir per line, slash-separated, with the filecap/dircap
+  separated by a space.
+
+  If output=JSON is added to the queryargs, then the results will be a
+  JSON-formatted list of (path, cap) tuples, where path is a list of strings.
 
 GET $DIRURL?t=deep-size
 

src/allmydata/dirnode.py

@@ -503,8 +503,8 @@ class NewDirectoryNode:
 
 
     def build_manifest(self):
-        """Return a frozenset of verifier-capability strings for all nodes
+        """Return a list of (path, cap) tuples, for all nodes (directories
-        (directories and files) reachable from this one."""
+        and files) reachable from this one."""
         return self.deep_traverse(ManifestWalker())
 
     def deep_stats(self):
@@ -521,17 +521,13 @@ class NewDirectoryNode:
 
 class ManifestWalker:
     def __init__(self):
-        self.manifest = set()
+        self.manifest = []
     def add_node(self, node, path):
-        v = node.get_verifier()
+        self.manifest.append( (tuple(path), node.get_uri()) )
-        # LIT files have no verify-cap, so don't add them
-        if v:
-            assert not isinstance(v, str), "ICK: %s %s" % (v, node)
-            self.manifest.add(v.to_string())
     def enter_directory(self, parent, children):
         pass
     def finish(self):
-        return frozenset(self.manifest)
+        return self.manifest
 
 
 class DeepStats:

src/allmydata/interfaces.py

@@ -793,9 +793,10 @@ class IDirectoryNode(IMutableFilesystemNode):
         operation finishes. The child name must be a unicode string."""
 
     def build_manifest():
-        """Return a Deferred that fires with a frozenset of
+        """Return a Deferred that fires with a list of (path, cap) tuples for
-        verifier-capability strings for all nodes (directories and files)
+        nodes (directories and files) reachable from this one. 'path' will be
-        reachable from this one."""
+        a tuple of unicode strings. The origin dirnode will be represented by
+        an empty path tuple."""
 
     def deep_stats():
         """Return a Deferred that fires with a dictionary of statistics

src/allmydata/test/test_dirnode.py

@@ -280,7 +280,7 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
             self.failUnless(u_ro.startswith("URI:DIR2-RO:"), u_ro)
             u_v = n.get_verifier().to_string()
             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
-            self.expected_manifest.append(u_v)
+            self.expected_manifest.append( ((), u) )
             expected_si = n._uri._filenode_uri.storage_index
             self.failUnlessEqual(n.get_storage_index(), expected_si)
 
@@ -292,7 +292,7 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
             other_file_uri = make_mutable_file_uri()
             m = Marker(fake_file_uri)
             ffu_v = m.get_verifier().to_string()
-            self.expected_manifest.append(ffu_v)
+            self.expected_manifest.append( ((u"child",) , m.get_uri()) )
             d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
             d.addCallback(lambda res:
                           self.shouldFail(ExistingChildError, "set_uri-no",
@@ -312,7 +312,7 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
                 self.subdir = subdir
                 new_v = subdir.get_verifier().to_string()
                 assert isinstance(new_v, str)
-                self.expected_manifest.append(new_v)
+                self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
             d.addCallback(_created)
 
             d.addCallback(lambda res:

src/allmydata/test/test_system.py

@@ -928,13 +928,14 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
 
             d1.addCallback(lambda res: home.build_manifest())
             d1.addCallback(self.log, "manifest")
-            #  four items:
+            #  five items:
+            # P/
             # P/personal/
             # P/personal/sekrit data
             # P/s2-rw  (same as P/s2-ro)
             # P/s2-rw/mydata992 (same as P/s2-rw/mydata992)
             d1.addCallback(lambda manifest:
-                           self.failUnlessEqual(len(manifest), 4))
+                           self.failUnlessEqual(len(manifest), 5))
             d1.addCallback(lambda res: home.deep_stats())
             def _check_stats(stats):
                 expected = {"count-immutable-files": 1,

src/allmydata/test/test_web.py

@@ -831,10 +831,33 @@ class Web(WebMixin, unittest.TestCase):
         return d
 
     def test_GET_DIRURL_manifest(self):
-        d = self.GET(self.public_url + "/foo?t=manifest", followRedirect=True)
+        def getman(ignored, suffix, followRedirect=False):
-        def _got(manifest):
+            return self.GET(self.public_url + "/foo" + suffix,
-            self.failUnless("Refresh Capabilities" in manifest)
+                            followRedirect=followRedirect)
-        d.addCallback(_got)
+        d = defer.succeed(None)
+        d.addCallback(getman, "?t=manifest", followRedirect=True)
+        def _got_html(manifest):
+            self.failUnless("Manifest of SI=" in manifest)
+            self.failUnless("<td>sub</td>" in manifest)
+            self.failUnless(self._sub_uri in manifest)
+            self.failUnless("<td>sub/baz.txt</td>" in manifest)
+        d.addCallback(_got_html)
+        d.addCallback(getman, "/?t=manifest")
+        d.addCallback(_got_html)
+        d.addCallback(getman, "/?t=manifest&output=text")
+        def _got_text(manifest):
+            self.failUnless("\nsub " + self._sub_uri + "\n" in manifest)
+            self.failUnless("\nsub/baz.txt URI:CHK:" in manifest)
+        d.addCallback(_got_text)
+        d.addCallback(getman, "/?t=manifest&output=JSON")
+        def _got_json(manifest):
+            data = simplejson.loads(manifest)
+            got = {}
+            for (path_list, cap) in data:
+                got[tuple(path_list)] = cap
+            self.failUnlessEqual(got[(u"sub",)], self._sub_uri)
+            self.failUnless((u"sub",u"baz.txt") in got)
+        d.addCallback(_got_json)
         return d
 
     def test_GET_DIRURL_deepsize(self):

src/allmydata/web/directory.py

@@ -6,7 +6,7 @@ import time
 from twisted.internet import defer
 from twisted.python.failure import Failure
 from twisted.web import http, html
-from nevow import url, rend, tags as T
+from nevow import url, rend, inevow, tags as T
 from nevow.inevow import IRequest
 
 from foolscap.eventual import fireEventually
@@ -676,6 +676,36 @@ class RenameForm(rend.Page):
 class Manifest(rend.Page):
     docFactory = getxmlfile("manifest.xhtml")
 
+    def renderHTTP(self, ctx):
+        output = get_arg(inevow.IRequest(ctx), "output", "html").lower()
+        if output == "text":
+            return self.text(ctx)
+        if output == "json":
+            return self.json(ctx)
+        return rend.Page.renderHTTP(self, ctx)
+
+    def slashify_path(self, path):
+        if not path:
+            return ""
+        return "/".join([p.encode("utf-8") for p in path])
+
+    def text(self, ctx):
+        inevow.IRequest(ctx).setHeader("content-type", "text/plain")
+        d = self.original.build_manifest()
+        def _render_text(manifest):
+            lines = []
+            for (path, cap) in manifest:
+                lines.append(self.slashify_path(path) + " " + cap)
+            return "\n".join(lines) + "\n"
+        d.addCallback(_render_text)
+        return d
+
+    def json(self, ctx):
+        inevow.IRequest(ctx).setHeader("content-type", "text/plain")
+        d = self.original.build_manifest()
+        d.addCallback(lambda manifest: simplejson.dumps(manifest))
+        return d
+
     def render_title(self, ctx):
         return T.title["Manifest of SI=%s" % abbreviated_dirnode(self.original)]
 
@@ -685,8 +715,9 @@ class Manifest(rend.Page):
     def data_items(self, ctx, data):
         return self.original.build_manifest()
 
-    def render_row(self, ctx, refresh_cap):
+    def render_row(self, ctx, (path, cap)):
-        ctx.fillSlots("refresh_capability", refresh_cap)
+        ctx.fillSlots("path", self.slashify_path(path))
+        ctx.fillSlots("cap", cap)
         return ctx.tag
 
 def DeepSize(ctx, dirnode):

src/allmydata/web/info.py

@@ -230,6 +230,13 @@ class MoreInfo(rend.Page):
             T.fieldset[
             T.input(type="hidden", name="t", value="manifest"),
             T.legend(class_="freeform-form-label")["Run a manifest operation (EXPENSIVE)"],
+            T.div["Output Format: ",
+                  T.select(name="output")
+                  [ T.option(value="html", selected="true")["HTML"],
+                    T.option(value="text")["text"],
+                    T.option(value="json")["JSON"],
+                    ],
+                  ],
             T.input(type="submit", value="Manifest"),
             ]]
         return ctx.tag[manifest]

src/allmydata/web/manifest.xhtml

@@ -13,10 +13,12 @@
 
 <table n:render="sequence" n:data="items" border="1">
   <tr n:pattern="header">
-    <td>Refresh Capabilities</td>
+    <td>Path</td>
+    <td>cap</td>
   </tr>
   <tr n:pattern="item" n:render="row">
-    <td><n:slot name="refresh_capability"/></td>
+    <td><n:slot name="path"/></td>
+    <td><n:slot name="cap"/></td>
   </tr>
 
   <tr n:pattern="empty"><td>no items in the manifest!</td></tr>