CLI: modify 'tahoe manifest' and 'tahoe deep-check' to report ERROR: properly. For #590.

src/allmydata/scripts/tahoe_check.py

@@ -89,16 +89,28 @@ class FakeTransport:
 
 class DeepCheckOutput(LineOnlyReceiver):
     delimiter = "\n"
-    def __init__(self, options):
+    def __init__(self, streamer, options):
+        self.streamer = streamer
         self.transport = FakeTransport()
 
         self.verbose = bool(options["verbose"])
         self.stdout = options.stdout
+        self.stderr = options.stderr
         self.num_objects = 0
         self.files_healthy = 0
         self.files_unhealthy = 0
+        self.in_error = False
 
     def lineReceived(self, line):
+        if self.in_error:
+            print >>self.stderr, line
+            return
+        if line.startswith("ERROR:"):
+            self.in_error = True
+            self.streamer.rc = 1
+            print >>self.stderr, line
+            return
+
         d = simplejson.loads(line)
         stdout = self.stdout
         if d["type"] not in ("file", "directory"):
@@ -131,17 +143,21 @@ class DeepCheckOutput(LineOnlyReceiver):
                   (serverid, storage_index, sharenum)
 
     def done(self):
+        if self.in_error:
+            return
         stdout = self.stdout
         print >>stdout, "done: %d objects checked, %d healthy, %d unhealthy" \
               % (self.num_objects, self.files_healthy, self.files_unhealthy)
 
 class DeepCheckAndRepairOutput(LineOnlyReceiver):
     delimiter = "\n"
-    def __init__(self, options):
+    def __init__(self, streamer, options):
+        self.streamer = streamer
         self.transport = FakeTransport()
 
         self.verbose = bool(options["verbose"])
         self.stdout = options.stdout
+        self.stderr = options.stderr
         self.num_objects = 0
         self.pre_repair_files_healthy = 0
         self.pre_repair_files_unhealthy = 0
@@ -149,8 +165,18 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver):
         self.repairs_successful = 0
         self.post_repair_files_healthy = 0
         self.post_repair_files_unhealthy = 0
+        self.in_error = False
 
     def lineReceived(self, line):
+        if self.in_error:
+            print >>self.stderr, line
+            return
+        if line.startswith("ERROR:"):
+            self.in_error = True
+            self.streamer.rc = 1
+            print >>self.stderr, line
+            return
+
         d = simplejson.loads(line)
         stdout = self.stdout
         if d["type"] not in ("file", "directory"):
@@ -211,6 +237,8 @@ class DeepCheckAndRepairOutput(LineOnlyReceiver):
                 print >>stdout, " repair failed"
 
     def done(self):
+        if self.in_error:
+            return
         stdout = self.stdout
         print >>stdout, "done: %d objects checked" % self.num_objects
         print >>stdout, " pre-repair: %d healthy, %d unhealthy" \
@@ -229,6 +257,7 @@ class DeepCheckStreamer(LineOnlyReceiver):
     def run(self, options):
         stdout = options.stdout
         stderr = options.stderr
+        self.rc = 0
         self.options = options
         nodeurl = options['node-url']
         if not nodeurl.endswith("/"):
@@ -247,9 +276,9 @@ class DeepCheckStreamer(LineOnlyReceiver):
             url += "&verify=true"
         if options["repair"]:
             url += "&repair=true"
-            output = DeepCheckAndRepairOutput(options)
+            output = DeepCheckAndRepairOutput(self, options)
         else:
-            output = DeepCheckOutput(options)
+            output = DeepCheckOutput(self, options)
         if options["add-lease"]:
             url += "&add-lease=true"
         resp = do_http("POST", url)
@@ -268,7 +297,7 @@ class DeepCheckStreamer(LineOnlyReceiver):
                 output.dataReceived(chunk)
         if not self.options["raw"]:
             output.done()
-        return 0
+        return self.rc
 
 def deepcheck(options):
     return DeepCheckStreamer().run(options)

src/allmydata/scripts/tahoe_manifest.py

@@ -16,6 +16,7 @@ class ManifestStreamer(LineOnlyReceiver):
         self.transport = FakeTransport()
 
     def run(self, options):
+        self.rc = 0
         stdout = options.stdout
         stderr = options.stderr
         self.options = options
@@ -38,6 +39,7 @@ class ManifestStreamer(LineOnlyReceiver):
             return 1
         #print "RESP", dir(resp)
         # use Twisted to split this into lines
+        self.in_error = False
         while True:
             chunk = resp.read(100)
             if not chunk:
@@ -46,11 +48,21 @@ class ManifestStreamer(LineOnlyReceiver):
                 stdout.write(chunk)
             else:
                 self.dataReceived(chunk)
-        return 0
+        return self.rc
 
     def lineReceived(self, line):
-        d = simplejson.loads(line)
         stdout = self.options.stdout
+        stderr = self.options.stderr
+        if self.in_error:
+            print >>stderr, line
+            return
+        if line.startswith("ERROR:"):
+            self.in_error = True
+            self.rc = 1
+            print >>stderr, line
+            return
+
+        d = simplejson.loads(line)
         if d["type"] in ("file", "directory"):
             if self.options["storage-index"]:
                 si = d["storage-index"]

src/allmydata/test/test_cli.py

@@ -1198,6 +1198,7 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
         d.addCallback(_stash_root_and_create_file)
         def _stash_uri(fn, which):
             self.uris[which] = fn.get_uri()
+            return fn
         d.addCallback(_stash_uri, "good")
         d.addCallback(lambda ign:
                       self.rootnode.add_file(u"small",
@@ -1217,6 +1218,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
                             in lines, out)
         d.addCallback(_check1)
 
+        # root
+        # root/good
+        # root/small
+        # root/mutable
+
         d.addCallback(lambda ign: self.do_cli("deep-check", "--verbose",
                                               self.rooturi))
         def _check2((rc, out, err)):
@@ -1248,6 +1254,11 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             debug.corrupt_share(cso)
         d.addCallback(_clobber_shares)
 
+        # root
+        # root/good  [9 shares]
+        # root/small
+        # root/mutable [1 corrupt share]
+
         d.addCallback(lambda ign:
                       self.do_cli("deep-check", "--verbose", self.rooturi))
         def _check3((rc, out, err)):
@@ -1315,5 +1326,55 @@ class Check(GridTestMixin, CLITestMixin, unittest.TestCase):
             self.failUnless(" post-repair: 4 healthy, 0 unhealthy" in lines,out)
         d.addCallback(_check6)
 
+        # now add a subdir, and a file below that, then make the subdir
+        # unrecoverable
+
+        d.addCallback(lambda ign:
+                      self.rootnode.create_empty_directory(u"subdir"))
+        d.addCallback(_stash_uri, "subdir")
+        d.addCallback(lambda fn:
+                      fn.add_file(u"subfile", upload.Data(DATA+"2", "")))
+        d.addCallback(lambda ign:
+                      self.delete_shares_numbered(self.uris["subdir"],
+                                                  range(10)))
+
+        # root
+        # root/good
+        # root/small
+        # root/mutable
+        # root/subdir [unrecoverable: 0 shares]
+        # root/subfile
+
+        d.addCallback(lambda ign: self.do_cli("manifest", self.rooturi))
+        def _manifest_failed((rc, out, err)):
+            self.failIfEqual(rc, 0)
+            self.failUnlessIn("ERROR: UnrecoverableFileError", err)
+            # the fatal directory should still show up, as the last line
+            self.failUnlessIn(" subdir\n", out)
+        d.addCallback(_manifest_failed)
+
+        d.addCallback(lambda ign: self.do_cli("deep-check", self.rooturi))
+        def _deep_check_failed((rc, out, err)):
+            self.failIfEqual(rc, 0)
+            self.failUnlessIn("ERROR: UnrecoverableFileError", err)
+            # we want to make sure that the error indication is the last
+            # thing that gets emitted
+            self.failIf("done:" in out, out)
+        d.addCallback(_deep_check_failed)
+
+        # this test is disabled until the deep-repair response to an
+        # unrepairable directory is fixed. The failure-to-repair should not
+        # throw an exception, but the failure-to-traverse that follows
+        # should throw UnrecoverableFileError.
+
+        #d.addCallback(lambda ign:
+        #              self.do_cli("deep-check", "--repair", self.rooturi))
+        #def _deep_check_repair_failed((rc, out, err)):
+        #    self.failIfEqual(rc, 0)
+        #    print err
+        #    self.failUnlessIn("ERROR: UnrecoverableFileError", err)
+        #    self.failIf("done:" in out, out)
+        #d.addCallback(_deep_check_repair_failed)
+
         return d