manifest: include stats in results. webapi is unchanged.

docs/frontends/webapi.txt

@@ -964,11 +964,13 @@ POST $DIRURL?t=start-manifest    (must add &ophandle=XYZ)
   by a space.
 
   If output=JSON is added to the queryargs, then the results will be a
-  JSON-formatted dictionary with three keys:
+  JSON-formatted dictionary with four keys:
 
    finished (bool): if False then you must reload the page until True
    origin_si (str): the storage index of the starting point
    manifest: list of (path, cap) tuples, where path is a list of strings.
+   stats: a dictionary with the same keys as the t=deep-stats command
+          (described below)
 
 POST $DIRURL?t=start-deep-size    (must add &ophandle=XYZ)
 

src/allmydata/dirnode.py

@@ -536,21 +536,6 @@ class NewDirectoryNode:
         return self.deep_traverse(DeepChecker(self, verify, repair=True))
 
 
-class ManifestWalker:
-    def __init__(self, origin):
-        self.manifest = []
-        self.origin = origin
-    def set_monitor(self, monitor):
-        self.monitor = monitor
-        monitor.origin_si = self.origin.get_storage_index()
-        monitor.set_status(self.manifest)
-    def add_node(self, node, path):
-        self.manifest.append( (tuple(path), node.get_uri()) )
-    def enter_directory(self, parent, children):
-        pass
-    def finish(self):
-        return self.manifest
-
 
 class DeepStats:
     def __init__(self, origin):
@@ -651,6 +636,20 @@ class DeepStats:
     def finish(self):
         return self.get_results()
 
+class ManifestWalker(DeepStats):
+    def __init__(self, origin):
+        DeepStats.__init__(self, origin)
+        self.manifest = []
+
+    def add_node(self, node, path):
+        self.manifest.append( (tuple(path), node.get_uri()) )
+        return DeepStats.add_node(self, node, path)
+
+    def finish(self):
+        return {"manifest": self.manifest,
+                "stats": self.get_results(),
+                }
+
 
 class DeepChecker:
     def __init__(self, root, verify, repair):

src/allmydata/interfaces.py

@@ -871,12 +871,18 @@ class IDirectoryNode(IMutableFilesystemNode):
         NoSuchChildError if I do not have a child by that name."""
 
     def build_manifest():
-        """Return a Monitor. The Monitor's results will be a list of (path,
+        """I generate a table of everything reachable from this directory.
-        cap) tuples for nodes (directories and files) reachable from this
+        I also compute deep-stats as described below.
-        one. 'path' will be a tuple of unicode strings. The origin dirnode
+
-        will be represented by an empty path tuple. The Monitor will also
+        I return a Monitor. The Monitor's results will be a dictionary with
-        have an .origin_si attribute with the (binary) storage index of the
+        two elements. The 'manifest' element is a list of (path, cap) tuples
-        starting point.
+        for nodes (directories and files) reachable from this one. 'path'
+        will be a tuple of unicode strings. The origin dirnode will be
+        represented by an empty path tuple. The 'stats' element is a
+        dictionary, the same that is generated by start_deep_stats() below.
+
+        The Monitor will also have an .origin_si attribute with the (binary)
+        storage index of the starting point.
         """
 
     def start_deep_stats():

src/allmydata/test/test_dirnode.py

@@ -335,12 +335,6 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
                           self.failUnlessEqual(sorted(children.keys()),
                                                sorted([u"child", u"subdir"])))
 
-            d.addCallback(lambda res: n.build_manifest().when_done())
-            def _check_manifest(manifest):
-                self.failUnlessEqual(sorted(manifest),
-                                     sorted(self.expected_manifest))
-            d.addCallback(_check_manifest)
-
             d.addCallback(lambda res: n.start_deep_stats().when_done())
             def _check_deepstats(stats):
                 self.failUnless(isinstance(stats, dict))
@@ -367,6 +361,15 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
                 self.failUnlessEqual(stats["size-files-histogram"], [])
             d.addCallback(_check_deepstats)
 
+            d.addCallback(lambda res: n.build_manifest().when_done())
+            def _check_manifest(res):
+                manifest = res["manifest"]
+                self.failUnlessEqual(sorted(manifest),
+                                     sorted(self.expected_manifest))
+                stats = res["stats"]
+                _check_deepstats(stats)
+            d.addCallback(_check_manifest)
+
             def _add_subsubdir(res):
                 return self.subdir.create_empty_directory(u"subsubdir")
             d.addCallback(_add_subsubdir)

src/allmydata/test/test_system.py

@@ -684,8 +684,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
             d1.addCallback(lambda res: dnode.has_child(u"see recursive"))
             d1.addCallback(lambda answer: self.failUnlessEqual(answer, True))
             d1.addCallback(lambda res: dnode.build_manifest().when_done())
-            d1.addCallback(lambda manifest:
+            d1.addCallback(lambda res:
-                           self.failUnlessEqual(len(manifest), 1))
+                           self.failUnlessEqual(len(res["manifest"]), 1))
             return d1
         d.addCallback(_created_dirnode)
 
@@ -975,8 +975,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
             # P/personal/sekrit data
             # P/s2-rw  (same as P/s2-ro)
             # P/s2-rw/mydata992 (same as P/s2-rw/mydata992)
-            d1.addCallback(lambda manifest:
+            d1.addCallback(lambda res:
-                           self.failUnlessEqual(len(manifest), 5))
+                           self.failUnlessEqual(len(res["manifest"]), 5))
             d1.addCallback(lambda res: home.start_deep_stats().when_done())
             def _check_stats(stats):
                 expected = {"count-immutable-files": 1,

src/allmydata/web/directory.py

@@ -724,14 +724,15 @@ class ManifestResults(rend.Page, ReloadMixin):
         lines = []
         is_finished = self.monitor.is_finished()
         lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
-        for (path, cap) in self.monitor.get_status():
+        for (path, cap) in self.monitor.get_status()["manifest"]:
             lines.append(self.slashify_path(path) + " " + cap)
         return "\n".join(lines) + "\n"
 
     def json(self, ctx):
         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
         m = self.monitor
-        status = {"manifest": m.get_status(),
+        status = {"manifest": m.get_status()["manifest"],
+                  "stats": m.get_status()["stats"],
                   "finished": m.is_finished(),
                   "origin": base32.b2a(m.origin_si),
                   }
@@ -747,7 +748,7 @@ class ManifestResults(rend.Page, ReloadMixin):
         return T.p["Manifest of SI=%s" % self._si_abbrev()]
 
     def data_items(self, ctx, data):
-        return self.monitor.get_status()
+        return self.monitor.get_status()["manifest"]
 
     def render_row(self, ctx, (path, cap)):
         ctx.fillSlots("path", self.slashify_path(path))