CLI: add 'tahoe manifest', which takes a directory and returns a list of things you can reach from it

docs/CLI.txt

@@ -325,6 +325,21 @@ tahoe mv tahoe:uploaded.txt fun:uploaded.txt
  These move a file from your tahoe root directory to the virtual directory
  set up earlier with "tahoe add-alias fun DIRCAP"
 
+== Virtual Drive Maintenance ==
+
+tahoe manifest tahoe:
+tahoe manifest --storage-index tahoe:
+
+ This performs a recursive walk of the given directory, visiting every file
+ and directory that can be reached from that point. It then emits one line to
+ stdout for each object it encounters.
+
+ The default behavior is to print the access cap string (like URI:CHK:.. or
+ URI:DIR2:..), followed by a space, followed by the full path name.
+
+ If --storage-index is added, each line will instead contain the object's
+ storage index. This (string) value is useful to determine which share files
+ (on the server) are associated with this directory tree.
 
 
 == Debugging ==

src/allmydata/scripts/cli.py

@@ -197,6 +197,18 @@ class WebopenOptions(VDriveOptions):
 
     longdesc = """Opens a webbrowser to the contents of some portion of the virtual drive."""
 
+class ManifestOptions(VDriveOptions):
+    optFlags = [
+        ("storage-index", "s", "Only print storage index strings, not pathname+cap"),
+        ]
+    def parseArgs(self, where=''):
+        self.where = where
+
+    def getSynopsis(self):
+        return "%s manifest [ALIAS:PATH]" % (os.path.basename(sys.argv[0]),)
+
+    longdesc = """Print a list of all files/directories reachable from the given starting point."""
+
 subCommands = [
     ["mkdir", None, MakeDirectoryOptions, "Create a new directory"],
     ["add-alias", None, AddAliasOptions, "Add a new alias cap"],
@@ -210,6 +222,7 @@ subCommands = [
     ["mv", None, MvOptions, "Move a file within the virtual drive."],
     ["ln", None, LnOptions, "Make an additional link to an existing file."],
     ["webopen", None, WebopenOptions, "Open a webbrowser to the root_dir"],
+    ["manifest", None, ManifestOptions, "List all files/dirs in a subtree"],
     ]
 
 def mkdir(options):
@@ -281,6 +294,11 @@ def webopen(options, opener=None):
     rc = tahoe_webopen.webopen(options, opener=opener)
     return rc
 
+def manifest(options):
+    from allmydata.scripts import tahoe_manifest
+    rc = tahoe_manifest.manifest(options)
+    return rc
+
 dispatch = {
     "mkdir": mkdir,
     "add-alias": add_alias,
@@ -294,5 +312,6 @@ dispatch = {
     "mv": mv,
     "ln": ln,
     "webopen": webopen,
+    "manifest": manifest,
     }
 

src/allmydata/scripts/tahoe_manifest.py

@@ -0,0 +1,91 @@
+
+import os, time
+from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
+from allmydata.scripts.common_http import do_http
+from allmydata.util import base32
+from allmydata import uri
+import urllib
+import simplejson
+
+class ManifestGrabber:
+    def run(self, options):
+        stderr = options.stderr
+        self.options = options
+        self.ophandle = ophandle = base32.b2a(os.urandom(16))
+        nodeurl = options['node-url']
+        if not nodeurl.endswith("/"):
+            nodeurl += "/"
+        self.nodeurl = nodeurl
+        where = options.where
+        rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
+        if path == '/':
+            path = ''
+        url = nodeurl + "uri/%s" % urllib.quote(rootcap)
+        if path:
+            url += "/" + escape_path(path)
+        # todo: should it end with a slash?
+        url += "?t=start-manifest&ophandle=" + ophandle
+        resp = do_http("POST", url)
+        if resp.status not in (200, 302):
+            print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
+            return 1
+        # now we poll for results. We nominally poll at t=1, 5, 10, 30, 60,
+        # 90, k*120 seconds, but if the poll takes non-zero time, that will
+        # be slightly longer. I'm not worried about trying to make up for
+        # that time.
+
+        return self.wait_for_results()
+
+    def poll_times(self):
+        for i in (1,5,10,30,60,90):
+            yield i
+        i = 120
+        while True:
+            yield i
+            i += 120
+
+    def wait_for_results(self):
+        last = 0
+        for next in self.poll_times():
+            delay = next - last
+            time.sleep(delay)
+            last = next
+            if self.poll():
+                return 0
+
+    def poll(self):
+        url = self.nodeurl + "operations/" + self.ophandle
+        url += "?t=status&output=JSON&release-after-complete=true"
+        stdout = self.options.stdout
+        stderr = self.options.stderr
+        resp = do_http("GET", url)
+        if resp.status != 200:
+            print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
+            return True
+        data = simplejson.loads(resp.read())
+        if not data["finished"]:
+            return False
+        self.write_results(data)
+        return True
+
+    def write_results(self, data):
+        stdout = self.options.stdout
+        stderr = self.options.stderr
+        if self.options["storage-index"]:
+            for (path, cap) in data["manifest"]:
+                u = uri.from_string(str(cap))
+                si = u.get_storage_index()
+                if si is not None:
+                    print >>stdout, base32.b2a(si)
+        else:
+            for (path, cap) in data["manifest"]:
+                try:
+                    print >>stdout, cap, "/".join(path)
+                except UnicodeEncodeError:
+                    print >>stdout, cap, "/".join([p.encode("utf-8")
+                                                   for p in path])
+
+
+
+def manifest(options):
+    return ManifestGrabber().run(options)

src/allmydata/test/test_system.py

@@ -2068,6 +2068,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         d.addCallback(self.do_stats)
         d.addCallback(self.do_test_check_good)
         d.addCallback(self.do_test_web_good)
+        d.addCallback(self.do_test_cli_good)
         d.addErrback(self.explain_web_error)
         d.addErrback(self.explain_error)
         return d
@@ -2345,6 +2346,50 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
         return d
 
+    def _run_cli(self, argv, stdin=""):
+        #print "CLI:", argv
+        stdout, stderr = StringIO(), StringIO()
+        d = threads.deferToThread(runner.runner, argv, run_by_human=False,
+                                  stdin=StringIO(stdin),
+                                  stdout=stdout, stderr=stderr)
+        def _done(res):
+            return stdout.getvalue(), stderr.getvalue()
+        d.addCallback(_done)
+        return d
+
+    def do_test_cli_good(self, ignored):
+        d = self._run_cli(["manifest", "-u", self.webish_url, self.root_uri])
+        def _check((out,err)):
+            lines = [l for l in out.split("\n") if l]
+            self.failUnlessEqual(len(lines), 4)
+            caps = {}
+            for l in lines:
+                try:
+                    cap, path = l.split(None, 1)
+                except ValueError:
+                    cap = l.strip()
+                    path = ""
+                caps[cap] = path
+            self.failUnless(self.root.get_uri() in caps)
+            self.failUnlessEqual(caps[self.root.get_uri()], "")
+            self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
+            self.failUnlessEqual(caps[self.large.get_uri()], "large")
+            self.failUnlessEqual(caps[self.small.get_uri()], "small")
+        d.addCallback(_check)
+
+        d.addCallback(lambda res:
+                      self._run_cli(["manifest", "-u", self.webish_url,
+                                     "--storage-index", self.root_uri]))
+        def _check2((out,err)):
+            lines = [l for l in out.split("\n") if l]
+            self.failUnlessEqual(len(lines), 3)
+            self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
+            self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
+            self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
+        d.addCallback(_check2)
+        return d
+
+
 class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 
     def test_bad(self):

src/allmydata/uri.py

@@ -38,6 +38,9 @@ class _BaseURI:
     def to_human_encoding(self):
         return 'http://127.0.0.1:8123/uri/'+self.to_string()
 
+    def get_storage_index(self):
+        return self.storage_index
+
 class CHKFileURI(_BaseURI):
     implements(IURI, IFileURI)
 
@@ -179,6 +182,8 @@ class LiteralFileURI(_BaseURI):
         return False
     def get_readonly(self):
         return self
+    def get_storage_index(self):
+        return None
 
     def get_verifier(self):
         # LIT files need no verification, all the data is present in the URI
@@ -359,6 +364,9 @@ class _NewDirectoryBaseURI(_BaseURI):
     def get_verifier(self):
         return NewDirectoryURIVerifier(self._filenode_uri.get_verifier())
 
+    def get_storage_index(self):
+        return self._filenode_uri.get_storage_index()
+
 class NewDirectoryURI(_NewDirectoryBaseURI):
     implements(INewDirectoryURI)