dirnode manifest: add verifycaps, both to internal API and to webapi. This will give the manual-GC tools more to work with, so they can estimate how much space will be freed.

docs/CLI.txt

@@ -329,6 +329,7 @@ tahoe mv tahoe:uploaded.txt fun:uploaded.txt
 
 tahoe manifest tahoe:
 tahoe manifest --storage-index tahoe:
+tahoe manifest --raw tahoe:
 
  This performs a recursive walk of the given directory, visiting every file
  and directory that can be reached from that point. It then emits one line to
@@ -339,7 +340,9 @@ tahoe manifest --storage-index tahoe:
 
  If --storage-index is added, each line will instead contain the object's
  storage index. This (string) value is useful to determine which share files
- (on the server) are associated with this directory tree.
+ (on the server) are associated with this directory tree. If --raw is
+ provided instead, the output will be a JSON-encoded dictionary that includes
+ keys for storage index strings, verifycaps, and deep-stats.
 
 tahoe stats tahoe:
 

docs/frontends/webapi.txt

@@ -964,11 +964,12 @@ POST $DIRURL?t=start-manifest    (must add &ophandle=XYZ)
   by a space.
 
   If output=JSON is added to the queryargs, then the results will be a
-  JSON-formatted dictionary with five keys:
+  JSON-formatted dictionary with six keys:
 
    finished (bool): if False then you must reload the page until True
    origin_si (base32 str): the storage index of the starting point
    manifest: list of (path, cap) tuples, where path is a list of strings.
+   verifycaps: list of (printable) verify cap strings
    storage-index: list of (base32) storage index strings
    stats: a dictionary with the same keys as the t=deep-stats command
           (described below)

src/allmydata/dirnode.py

@@ -641,17 +641,22 @@ class ManifestWalker(DeepStats):
         DeepStats.__init__(self, origin)
         self.manifest = []
         self.storage_index_strings = set()
+        self.verifycaps = set()
 
     def add_node(self, node, path):
         self.manifest.append( (tuple(path), node.get_uri()) )
         si = node.get_storage_index()
         if si:
             self.storage_index_strings.add(base32.b2a(si))
+        v = node.get_verifier()
+        if v:
+            self.verifycaps.add(v.to_string())
         return DeepStats.add_node(self, node, path)
 
     def get_results(self):
         stats = DeepStats.get_results(self)
         return {"manifest": self.manifest,
+                "verifycaps": self.verifycaps,
                 "storage-index": self.storage_index_strings,
                 "stats": stats,
                 }

src/allmydata/interfaces.py

@@ -863,16 +863,19 @@ class IDirectoryNode(IMutableFilesystemNode):
         I also compute deep-stats as described below.
 
         I return a Monitor. The Monitor's results will be a dictionary with
-        three elements:
+        four elements:
 
          res['manifest']: a list of (path, cap) tuples for all nodes
                           (directories and files) reachable from this one.
                           'path' will be a tuple of unicode strings. The
                           origin dirnode will be represented by an empty path
                           tuple.
+         res['verifycaps']: a list of (printable) verifycap strings, one for
+                            each reachable non-LIT node. This is a set:
+                            it will contain no duplicates.
          res['storage-index']: a list of (base32) storage index strings,
-                               one for each reachable node. This is a set:
+                               one for each reachable non-LIT node. This is
-                               duplicates have been removed.
+                               a set: it will contain no duplicates.
          res['stats']: a dictionary, the same that is generated by
                        start_deep_stats() below.
 

src/allmydata/test/test_dirnode.py

@@ -9,7 +9,7 @@ from allmydata.interfaces import IURI, IClient, IMutableFileNode, \
      INewDirectoryURI, IReadonlyNewDirectoryURI, IFileNode, \
      ExistingChildError, NoSuchChildError, \
      IDeepCheckResults, IDeepCheckAndRepairResults
-from allmydata.util import hashutil
+from allmydata.util import hashutil, base32
 from allmydata.monitor import Monitor
 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
      FakeDirectoryNode, create_chk_filenode, ErrorMixin
@@ -281,6 +281,8 @@ class Dirnode(unittest.TestCase,
 
     def test_create(self):
         self.expected_manifest = []
+        self.expected_verifycaps = set()
+        self.expected_storage_indexes = set()
 
         d = self.client.create_empty_dirnode()
         def _then(n):
@@ -294,8 +296,11 @@ class Dirnode(unittest.TestCase,
             u_v = n.get_verifier().to_string()
             self.failUnless(u_v.startswith("URI:DIR2-Verifier:"), u_v)
             self.expected_manifest.append( ((), u) )
+            self.expected_verifycaps.add(u_v)
+            si = n.get_storage_index()
+            self.expected_storage_indexes.add(base32.b2a(si))
             expected_si = n._uri._filenode_uri.storage_index
-            self.failUnlessEqual(n.get_storage_index(), expected_si)
+            self.failUnlessEqual(si, expected_si)
 
             d = n.list()
             d.addCallback(lambda res: self.failUnlessEqual(res, {}))
@@ -306,6 +311,8 @@ class Dirnode(unittest.TestCase,
             m = Marker(fake_file_uri)
             ffu_v = m.get_verifier().to_string()
             self.expected_manifest.append( ((u"child",) , m.get_uri()) )
+            self.expected_verifycaps.add(ffu_v)
+            self.expected_storage_indexes.add(base32.b2a(m.get_storage_index()))
             d.addCallback(lambda res: n.set_uri(u"child", fake_file_uri))
             d.addCallback(lambda res:
                           self.shouldFail(ExistingChildError, "set_uri-no",
@@ -326,6 +333,9 @@ class Dirnode(unittest.TestCase,
                 new_v = subdir.get_verifier().to_string()
                 assert isinstance(new_v, str)
                 self.expected_manifest.append( ((u"subdir",), subdir.get_uri()) )
+                self.expected_verifycaps.add(new_v)
+                si = subdir.get_storage_index()
+                self.expected_storage_indexes.add(base32.b2a(si))
             d.addCallback(_created)
 
             d.addCallback(lambda res:
@@ -372,6 +382,10 @@ class Dirnode(unittest.TestCase,
                                      sorted(self.expected_manifest))
                 stats = res["stats"]
                 _check_deepstats(stats)
+                self.failUnlessEqual(self.expected_verifycaps,
+                                     res["verifycaps"])
+                self.failUnlessEqual(self.expected_storage_indexes,
+                                     res["storage-index"])
             d.addCallback(_check_manifest)
 
             def _add_subsubdir(res):

src/allmydata/test/test_system.py

@@ -2423,6 +2423,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
                                      "--node-directory", basedir,
                                      "--storage-index", self.root_uri]))
         def _check2((out,err)):
+            self.failUnlessEqual(err, "")
             lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 3)
             self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
@@ -2430,6 +2431,28 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
             self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
         d.addCallback(_check2)
 
+        d.addCallback(lambda res:
+                      self._run_cli(["manifest",
+                                     "--node-directory", basedir,
+                                     "--raw", self.root_uri]))
+        def _check2r((out,err)):
+            self.failUnlessEqual(err, "")
+            data = simplejson.loads(out)
+            sis = data["storage-index"]
+            self.failUnlessEqual(len(sis), 3)
+            self.failUnless(base32.b2a(self.root.get_storage_index()) in sis)
+            self.failUnless(base32.b2a(self.mutable.get_storage_index()) in sis)
+            self.failUnless(base32.b2a(self.large.get_storage_index()) in sis)
+            self.failUnlessEqual(data["stats"]["count-files"], 4)
+            self.failUnlessEqual(data["origin"],
+                                 base32.b2a(self.root.get_storage_index()))
+            verifycaps = data["verifycaps"]
+            self.failUnlessEqual(len(verifycaps), 3)
+            self.failUnless(self.root.get_verifier().to_string() in verifycaps)
+            self.failUnless(self.mutable.get_verifier().to_string() in verifycaps)
+            self.failUnless(self.large.get_verifier().to_string() in verifycaps)
+        d.addCallback(_check2r)
+
         d.addCallback(lambda res:
                       self._run_cli(["stats",
                                      "--node-directory", basedir,

src/allmydata/test/test_web.py

@@ -950,6 +950,7 @@ class Web(WebMixin, testutil.StallMixin, unittest.TestCase):
             self.failUnless("finished" in res)
             self.failUnless("origin" in res)
             self.failUnless("storage-index" in res)
+            self.failUnless("verifycaps" in res)
             self.failUnless("stats" in res)
         d.addCallback(_got_json)
         return d

src/allmydata/web/directory.py

@@ -731,9 +731,11 @@ class ManifestResults(rend.Page, ReloadMixin):
     def json(self, ctx):
         inevow.IRequest(ctx).setHeader("content-type", "text/plain")
         m = self.monitor
-        status = {"manifest": m.get_status()["manifest"],
+        s = m.get_status()
-                  "storage-index": list(m.get_status()["storage-index"]),
+        status = {"manifest": s["manifest"],
-                  "stats": m.get_status()["stats"],
+                  "verifycaps": list(s["verifycaps"]),
+                  "storage-index": list(s["storage-index"]),
+                  "stats": s["stats"],
                   "finished": m.is_finished(),
                   "origin": base32.b2a(m.origin_si),
                   }