Make internal state JSON-able for lease-crawler

src/allmydata/storage/expirer.py

@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    # We omit anything that might end up in pickle, just in case.
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, range, str, max, min  # noqa: F401
-
-import time, os, pickle, struct
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+import json
+import time
+import os
+import struct
 from allmydata.storage.crawler import ShareCrawler
 from allmydata.storage.shares import get_share_file
 from allmydata.storage.common import UnknownMutableContainerVersionError, \
@@ -95,9 +96,7 @@ class LeaseCheckingCrawler(ShareCrawler):
         if not os.path.exists(self.historyfile):
             history = {} # cyclenum -> dict
             with open(self.historyfile, "wb") as f:
-                # Newer protocols won't work in Python 2; when it is dropped,
-                # protocol v4 can be used (added in Python 3.4).
-                pickle.dump(history, f, protocol=2)
+                json.dump(history, f)
 
     def create_empty_cycle_dict(self):
         recovered = self.create_empty_recovered_dict()
@@ -142,7 +141,7 @@ class LeaseCheckingCrawler(ShareCrawler):
                     struct.error):
                 twlog.msg("lease-checker error processing %s" % sharefile)
                 twlog.err()
-                which = (storage_index_b32, shnum)
+                which = [storage_index_b32, shnum]
                 self.state["cycle-to-date"]["corrupt-shares"].append(which)
                 wks = (1, 1, 1, "unknown")
             would_keep_shares.append(wks)
@@ -212,7 +211,7 @@ class LeaseCheckingCrawler(ShareCrawler):
                 num_valid_leases_configured += 1
 
         so_far = self.state["cycle-to-date"]
-        self.increment(so_far["leases-per-share-histogram"], num_leases, 1)
+        self.increment(so_far["leases-per-share-histogram"], str(num_leases), 1)
         self.increment_space("examined", s, sharetype)
 
         would_keep_share = [1, 1, 1, sharetype]
@@ -291,12 +290,14 @@ class LeaseCheckingCrawler(ShareCrawler):
 
         start = self.state["current-cycle-start-time"]
         now = time.time()
-        h["cycle-start-finish-times"] = (start, now)
+        h["cycle-start-finish-times"] = [start, now]
         h["expiration-enabled"] = self.expiration_enabled
-        h["configured-expiration-mode"] = (self.mode,
-                                           self.override_lease_duration,
-                                           self.cutoff_date,
-                                           self.sharetypes_to_expire)
+        h["configured-expiration-mode"] = [
+            self.mode,
+            self.override_lease_duration,
+            self.cutoff_date,
+            self.sharetypes_to_expire,
+        ]
 
         s = self.state["cycle-to-date"]
 
@@ -315,15 +316,13 @@ class LeaseCheckingCrawler(ShareCrawler):
         h["space-recovered"] = s["space-recovered"].copy()
 
         with open(self.historyfile, "rb") as f:
-            history = pickle.load(f)
-        history[cycle] = h
+            history = json.load(f)
+        history[str(cycle)] = h
         while len(history) > 10:
-            oldcycles = sorted(history.keys())
-            del history[oldcycles[0]]
+            oldcycles = sorted(int(k) for k in history.keys())
+            del history[str(oldcycles[0])]
         with open(self.historyfile, "wb") as f:
-            # Newer protocols won't work in Python 2; when it is dropped,
-            # protocol v4 can be used (added in Python 3.4).
-            pickle.dump(history, f, protocol=2)
+            json.dump(history, f)
 
     def get_state(self):
         """In addition to the crawler state described in
@@ -393,7 +392,7 @@ class LeaseCheckingCrawler(ShareCrawler):
 
         state = ShareCrawler.get_state(self) # does a shallow copy
         with open(self.historyfile, "rb") as f:
-            history = pickle.load(f)
+            history = json.load(f)
         state["history"] = history
 
         if not progress["cycle-in-progress"]:
@@ -406,10 +405,12 @@ class LeaseCheckingCrawler(ShareCrawler):
         lah = so_far["lease-age-histogram"]
         so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
         so_far["expiration-enabled"] = self.expiration_enabled
-        so_far["configured-expiration-mode"] = (self.mode,
-                                                self.override_lease_duration,
-                                                self.cutoff_date,
-                                                self.sharetypes_to_expire)
+        so_far["configured-expiration-mode"] = [
+            self.mode,
+            self.override_lease_duration,
+            self.cutoff_date,
+            self.sharetypes_to_expire,
+        ]
 
         so_far_sr = so_far["space-recovered"]
         remaining_sr = {}

src/allmydata/test/test_storage_web.py

@@ -376,7 +376,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             self.failUnlessEqual(type(lah), list)
             self.failUnlessEqual(len(lah), 1)
             self.failUnlessEqual(lah, [ (0.0, DAY, 1) ] )
-            self.failUnlessEqual(so_far["leases-per-share-histogram"], {1: 1})
+            self.failUnlessEqual(so_far["leases-per-share-histogram"], {"1": 1})
             self.failUnlessEqual(so_far["corrupt-shares"], [])
             sr1 = so_far["space-recovered"]
             self.failUnlessEqual(sr1["examined-buckets"], 1)
@@ -427,9 +427,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             self.failIf("cycle-to-date" in s)
             self.failIf("estimated-remaining-cycle" in s)
             self.failIf("estimated-current-cycle" in s)
-            last = s["history"][0]
+            last = s["history"]["0"]
             self.failUnlessIn("cycle-start-finish-times", last)
-            self.failUnlessEqual(type(last["cycle-start-finish-times"]), tuple)
+            self.failUnlessEqual(type(last["cycle-start-finish-times"]), list)
             self.failUnlessEqual(last["expiration-enabled"], False)
             self.failUnlessIn("configured-expiration-mode", last)
 
@@ -437,9 +437,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             lah = last["lease-age-histogram"]
             self.failUnlessEqual(type(lah), list)
             self.failUnlessEqual(len(lah), 1)
-            self.failUnlessEqual(lah, [ (0.0, DAY, 6) ] )
+            self.failUnlessEqual(lah, [ [0.0, DAY, 6] ] )
 
-            self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
+            self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2})
             self.failUnlessEqual(last["corrupt-shares"], [])
 
             rec = last["space-recovered"]
@@ -587,12 +587,12 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             self.failUnlessEqual(count_leases(mutable_si_3), 1)
 
             s = lc.get_state()
-            last = s["history"][0]
+            last = s["history"]["0"]
 
             self.failUnlessEqual(last["expiration-enabled"], True)
             self.failUnlessEqual(last["configured-expiration-mode"],
-                                 ("age", 2000, None, ("mutable", "immutable")))
-            self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
+                                 ["age", 2000, None, ["mutable", "immutable"]])
+            self.failUnlessEqual(last["leases-per-share-histogram"], {"1": 2, "2": 2})
 
             rec = last["space-recovered"]
             self.failUnlessEqual(rec["examined-buckets"], 4)
@@ -731,14 +731,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             self.failUnlessEqual(count_leases(mutable_si_3), 1)
 
             s = lc.get_state()
-            last = s["history"][0]
+            last = s["history"]["0"]
 
             self.failUnlessEqual(last["expiration-enabled"], True)
             self.failUnlessEqual(last["configured-expiration-mode"],
-                                 ("cutoff-date", None, then,
-                                  ("mutable", "immutable")))
+                                 ["cutoff-date", None, then,
+                                  ["mutable", "immutable"]])
             self.failUnlessEqual(last["leases-per-share-histogram"],
-                                 {1: 2, 2: 2})
+                                 {"1": 2, "2": 2})
 
             rec = last["space-recovered"]
             self.failUnlessEqual(rec["examined-buckets"], 4)
@@ -924,8 +924,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
             s = lc.get_state()
             h = s["history"]
             self.failUnlessEqual(len(h), 10)
-            self.failUnlessEqual(max(h.keys()), 15)
-            self.failUnlessEqual(min(h.keys()), 6)
+            self.failUnlessEqual(max(int(k) for k in h.keys()), 15)
+            self.failUnlessEqual(min(int(k) for k in h.keys()), 6)
         d.addCallback(_check)
         return d
 
@@ -1014,7 +1014,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
         def _check(ignored):
             s = lc.get_state()
-            last = s["history"][0]
+            last = s["history"]["0"]
             rec = last["space-recovered"]
             self.failUnlessEqual(rec["configured-buckets"], 4)
             self.failUnlessEqual(rec["configured-shares"], 4)
@@ -1110,7 +1110,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
 
         def _after_first_cycle(ignored):
             s = lc.get_state()
-            last = s["history"][0]
+            last = s["history"]["0"]
             rec = last["space-recovered"]
             self.failUnlessEqual(rec["examined-buckets"], 5)
             self.failUnlessEqual(rec["examined-shares"], 3)

src/allmydata/web/storage.py

@@ -256,8 +256,8 @@ class StorageStatusElement(Element):
 
         if so_far["corrupt-shares"]:
             add("Corrupt shares:",
-                T.ul( (T.li( ["SI %s shnum %d" % corrupt_share
-                              for corrupt_share in so_far["corrupt-shares"] ]
+                T.ul( (T.li( ["SI %s shnum %d" % (si, shnum)
+                              for si, shnum in so_far["corrupt-shares"] ]
                              ))))
         return tag("Current cycle:", p)
 
@@ -267,7 +267,8 @@ class StorageStatusElement(Element):
         h = lc.get_state()["history"]
         if not h:
             return ""
-        last = h[max(h.keys())]
+        biggest = str(max(int(k) for k in h.keys()))
+        last = h[biggest]
 
         start, end = last["cycle-start-finish-times"]
         tag("Last complete cycle (which took %s and finished %s ago)"
@@ -290,8 +291,8 @@ class StorageStatusElement(Element):
 
         if last["corrupt-shares"]:
             add("Corrupt shares:",
-                T.ul( (T.li( ["SI %s shnum %d" % corrupt_share
-                              for corrupt_share in last["corrupt-shares"] ]
+                T.ul( (T.li( ["SI %s shnum %d" % (si, shnum)
+                              for si, shnum in last["corrupt-shares"] ]
                              ))))
 
         return tag(p)