checker: add information to results, add some deep-check tests, fix a bug in which unhealthy files were not counted

src/allmydata/immutable/checker.py

@@ -28,6 +28,9 @@ class Results:
     def is_healthy(self):
         return self.healthy
 
+    def get_storage_index(self):
+        return self.storage_index
+
     def get_storage_index_string(self):
         return self.storage_index_s
 
@@ -59,6 +62,7 @@ class DeepCheckResults:
         self.repairs_attempted = 0
         self.repairs_successful = 0
         self.problems = []
+        self.all_results = {}
         self.server_problems = {}
 
     def get_root_storage_index_string(self):
@@ -66,10 +70,11 @@ class DeepCheckResults:
 
     def add_check(self, r):
         self.objects_checked += 1
-        if r.is_healthy:
+        if r.is_healthy():
             self.objects_healthy += 1
         else:
             self.problems.append(r)
+        self.all_results[r.get_storage_index()] = r
 
     def add_repair(self, is_successful):
         self.repairs_attempted += 1
@@ -88,7 +93,8 @@ class DeepCheckResults:
         return self.server_problems
     def get_problems(self):
         return self.problems
-
+    def get_all_results(self):
+        return self.all_results
 
 class SimpleCHKFileChecker:
     """Return a list of (needed, total, found, sharemap), where sharemap maps

src/allmydata/interfaces.py

@@ -1478,8 +1478,10 @@ class ICheckerResults(Interface):
         """Return a bool, True if the file is fully healthy, False if it is
         damaged in any way."""
 
+    def get_storage_index():
+        """Return a string with the (binary) storage index."""
     def get_storage_index_string():
-        """Return a string with the abbreviated storage index."""
+        """Return a string with the (printable) abbreviated storage index."""
     def get_mutability_string():
         """Return a string with 'mutable' or 'immutable'."""
 
@@ -1524,6 +1526,9 @@ class IDeepCheckResults(Interface):
     def get_problems():
         """Return a list of ICheckerResults, one for each object that
         was not fully healthy."""
+    def get_all_results():
+        """Return a dict mapping storage_index (a binary string) to an
+        ICheckerResults instance, one for each object that was checked."""
 
 class IRepairable(Interface):
     def repair(checker_results):

src/allmydata/mutable/checker.py

@@ -212,6 +212,8 @@ class Results:
     def is_healthy(self):
         return self.healthy
 
+    def get_storage_index(self):
+        return self.storage_index
     def get_storage_index_string(self):
         return self.storage_index_s
 

src/allmydata/test/common.py

@@ -12,6 +12,8 @@ from allmydata.interfaces import IURI, IMutableFileNode, IFileNode, \
      FileTooLargeError, ICheckable
 from allmydata.immutable import checker
 from allmydata.immutable.encode import NotEnoughSharesError
+from allmydata.mutable.checker import Results as MutableCheckerResults
+from allmydata.mutable.common import CorruptShareError
 from allmydata.util import log, testutil, fileutil
 from allmydata.stats import PickleStatsGatherer
 from allmydata.key_generator import KeyGeneratorService
@@ -29,10 +31,12 @@ class FakeCHKFileNode:
     dictionary."""
     implements(IFileNode)
     all_contents = {}
+    bad_shares = {}
 
     def __init__(self, u, client):
         self.client = client
         self.my_uri = u.to_string()
+        self.storage_index = u.storage_index
 
     def get_uri(self):
         return self.my_uri
@@ -42,8 +46,13 @@ class FakeCHKFileNode:
         return IURI(self.my_uri).get_verifier()
     def check(self, verify=False, repair=False):
         r = checker.Results(None)
-        r.healthy = True
-        r.problems = []
+        is_bad = self.bad_shares.get(self.storage_index, None)
+        if is_bad:
+             r.healthy = False
+             r.problems = failure.Failure(CorruptShareError(is_bad))
+        else:
+             r.healthy = True
+             r.problems = []
         return defer.succeed(r)
     def is_mutable(self):
         return False
@@ -90,6 +99,7 @@ class FakeMutableFileNode:
     implements(IMutableFileNode, ICheckable)
     MUTABLE_SIZELIMIT = 10000
     all_contents = {}
+    bad_shares = {}
 
     def __init__(self, client):
         self.client = client
@@ -125,9 +135,16 @@ class FakeMutableFileNode:
         return self.storage_index
 
     def check(self, verify=False, repair=False):
-        r = checker.Results(None)
-        r.healthy = True
-        r.problems = []
+        r = MutableCheckerResults(self.storage_index)
+        is_bad = self.bad_shares.get(self.storage_index, None)
+        if is_bad:
+             r.healthy = False
+             r.problems = failure.Failure(CorruptShareError("peerid",
+                                                            0, # shnum
+                                                            is_bad))
+        else:
+             r.healthy = True
+             r.problems = []
         return defer.succeed(r)
 
     def deep_check(self, verify=False, repair=False):

src/allmydata/test/test_dirnode.py

@@ -159,6 +159,25 @@ class Dirnode(unittest.TestCase, testutil.ShouldFailMixin, testutil.StallMixin):
         d.addCallback(_check_results)
         return d
 
+    def _mark_file_bad(self, rootnode):
+        si = IURI(rootnode.get_uri())._filenode_uri.storage_index
+        rootnode._node.bad_shares[si] = "unhealthy"
+        return rootnode
+
+    def test_deepcheck_problems(self):
+        d = self._test_deepcheck_create()
+        d.addCallback(lambda rootnode: self._mark_file_bad(rootnode))
+        d.addCallback(lambda rootnode: rootnode.deep_check())
+        def _check_results(r):
+            self.failUnlessEqual(r.count_objects_checked(), 3)
+            self.failUnlessEqual(r.count_objects_healthy(), 2)
+            self.failUnlessEqual(r.count_repairs_attempted(), 0)
+            self.failUnlessEqual(r.count_repairs_successful(), 0)
+            self.failUnlessEqual(len(r.get_server_problems()), 0)
+            self.failUnlessEqual(len(r.get_problems()), 1)
+        d.addCallback(_check_results)
+        return d
+
     def test_readonly(self):
         fileuri = make_chk_file_uri(1234)
         filenode = self.client.create_node_from_uri(fileuri)

src/allmydata/web/checker_results.py

@@ -2,6 +2,7 @@
 from nevow import rend, inevow, tags as T
 from allmydata.web.common import getxmlfile, get_arg
 from allmydata.interfaces import ICheckerResults, IDeepCheckResults
+from allmydata.util import base32
 
 class CheckerResults(rend.Page):
     docFactory = getxmlfile("checker-results.xhtml")
@@ -48,6 +49,23 @@ class DeepCheckResults(rend.Page):
     def data_problems(self, ctx, data):
         for cr in self.r.get_problems():
             yield cr
+    def render_problem(self, ctx, data):
+        cr = data
+        text = cr.get_storage_index_string()
+        text += ": "
+        text += cr.status_report
+        return ctx.tag[text]
+
+    def data_all_objects(self, ctx, data):
+        r = self.r.get_all_results()
+        for storage_index in sorted(r.keys()):
+            yield r[storage_index]
+
+    def render_object(self, ctx, data):
+        r = data
+        ctx.fillSlots("storage_index", r.get_storage_index_string())
+        ctx.fillSlots("healthy", str(r.is_healthy()))
+        return ctx.tag
 
     def render_return(self, ctx, data):
         req = inevow.IRequest(ctx)

src/allmydata/web/deep-check-results.xhtml

@@ -18,7 +18,7 @@
 <h2>Problems:</h2>
 
 <ul n:render="sequence" n:data="problems">
-  <li n:pattern="item" />
+  <li n:pattern="item" n:render="problem"/>
   <li n:pattern="empty">None</li>
 </ul>
 
@@ -28,6 +28,24 @@
   <li>Repairs Successful: <span n:render="data" n:data="repairs_successful" /></li>
 </ul>
 
+<h2>Objects Checked</h2>
+<div>
+<table n:render="sequence" n:data="all_objects" border="1">
+  <tr n:pattern="header">
+    <td>Storage Index</td>
+    <td>Healthy?</td>
+  </tr>
+  <tr n:pattern="item" n:render="object">
+    <td><n:slot name="storage_index"/></td>
+    <td><n:slot name="healthy"/></td>
+  </tr>
+
+  <tr n:pattern="empty"><td>no objects?</td></tr>
+
+</table>
+</div>
+
+
 <div n:render="return" />
 
   </body>