expirer: add mode to expire only-mutable or only-immutable shares

src/allmydata/storage/expirer.py

@@ -59,6 +59,9 @@ class LeaseCheckingCrawler(ShareCrawler):
             assert isinstance(expiration_mode[1], int) # seconds
         elif self.mode[0] == "date-cutoff":
             assert isinstance(expiration_mode[1], int) # seconds-since-epoch
+        self.sharetypes_to_expire = ("mutable", "immutable")
+        if len(self.mode) > 2:
+            self.sharetypes_to_expire = self.mode[2]
         ShareCrawler.__init__(self, server, statefile)
 
     def add_initial_state(self):
@@ -145,6 +148,7 @@ class LeaseCheckingCrawler(ShareCrawler):
     def process_share(self, sharefilename):
         # first, find out what kind of a share it is
         sf = get_share_file(sharefilename)
+        sftype = sf.sharetype
         now = time.time()
         s = self.stat(sharefilename)
 
@@ -165,19 +169,23 @@ class LeaseCheckingCrawler(ShareCrawler):
                 num_valid_leases_original += 1
 
             #  expired-or-not according to our configured age limit
+            expired = False
             if self.mode[0] == "age":
                 age_limit = self.mode[1]
-                if age < age_limit:
-                    num_valid_leases_configured += 1
-                else:
-                    expired_leases_configured.append(li)
+                if age > age_limit:
+                    expired = True
             else:
                 assert self.mode[0] == "date-cutoff"
                 date_cutoff = self.mode[1]
-                if grant_renew_time > date_cutoff:
-                    num_valid_leases_configured += 1
-                else:
-                    expired_leases_configured.append(li)
+                if grant_renew_time < date_cutoff:
+                    expired = True
+            if sftype not in self.sharetypes_to_expire:
+                expired = False
+
+            if expired:
+                expired_leases_configured.append(li)
+            else:
+                num_valid_leases_configured += 1
 
         so_far = self.state["cycle-to-date"]
         self.increment(so_far["leases-per-share-histogram"], num_leases, 1)

src/allmydata/storage/immutable.py

@@ -37,6 +37,7 @@ from allmydata.storage.common import UnknownImmutableContainerVersionError, \
 
 class ShareFile:
     LEASE_SIZE = struct.calcsize(">L32s32sL")
+    sharetype = "immutable"
 
     def __init__(self, filename, max_size=None, create=False):
         """ If max_size is not None then I won't allow more than max_size to be written to me. If create=True and max_size must not be None. """

src/allmydata/storage/mutable.py

@@ -32,6 +32,7 @@ assert struct.calcsize("Q"), 8 # The struct module doc says that Q's are 8 bytes
 
 class MutableShareFile:
 
+    sharetype = "mutable"
     DATA_LENGTH_OFFSET = struct.calcsize(">32s20s32s")
     EXTRA_LEASE_OFFSET = DATA_LENGTH_OFFSET + 8
     HEADER_SIZE = struct.calcsize(">32s20s32sQQ") # doesn't include leases

src/allmydata/test/test_storage.py

@@ -1741,7 +1741,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
                 f = open(sf.home, 'rb+')
                 sf._write_lease_record(f, i, lease)
                 f.close()
-            return
+                return
         raise IndexError("unable to renew non-existent lease")
 
     def test_expire_age(self):
@@ -2023,6 +2023,118 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d.addCallback(_check_html)
         return d
 
+    def test_only_immutable(self):
+        basedir = "storage/LeaseCrawler/only_immutable"
+        fileutil.make_dirs(basedir)
+        now = time.time()
+        then = int(now - 2000)
+        ss = StorageServer(basedir, "\x00" * 20,
+                           expiration_enabled=True,
+                           expiration_mode=("date-cutoff",
+                                            then, ("immutable",)))
+        lc = ss.lease_checker
+        lc.slow_start = 0
+        webstatus = StorageStatus(ss)
+
+        self.make_shares(ss)
+        [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis
+        # set all leases to be expirable
+        new_expiration_time = now - 3000 + 31*24*60*60
+
+        def count_shares(si):
+            return len(list(ss._iter_share_files(si)))
+        def _get_sharefile(si):
+            return list(ss._iter_share_files(si))[0]
+        def count_leases(si):
+            return len(list(_get_sharefile(si).get_leases()))
+
+        sf0 = _get_sharefile(immutable_si_0)
+        self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time)
+        sf1 = _get_sharefile(immutable_si_1)
+        self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time)
+        self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time)
+        sf2 = _get_sharefile(mutable_si_2)
+        self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time)
+        sf3 = _get_sharefile(mutable_si_3)
+        self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time)
+        self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time)
+
+        ss.setServiceParent(self.s)
+        def _wait():
+            return bool(lc.get_state()["last-cycle-finished"] is not None)
+        d = self.poll(_wait)
+
+        def _after_first_cycle(ignored):
+            self.failUnlessEqual(count_shares(immutable_si_0), 0)
+            self.failUnlessEqual(count_shares(immutable_si_1), 0)
+            self.failUnlessEqual(count_shares(mutable_si_2), 1)
+            self.failUnlessEqual(count_leases(mutable_si_2), 1)
+            self.failUnlessEqual(count_shares(mutable_si_3), 1)
+            self.failUnlessEqual(count_leases(mutable_si_3), 2)
+        d.addCallback(_after_first_cycle)
+        d.addCallback(lambda ign: self.render1(webstatus))
+        def _check_html(html):
+            s = remove_tags(html)
+            self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s)
+        d.addCallback(_check_html)
+        return d
+
+    def test_only_mutable(self):
+        basedir = "storage/LeaseCrawler/only_mutable"
+        fileutil.make_dirs(basedir)
+        now = time.time()
+        then = int(now - 2000)
+        ss = StorageServer(basedir, "\x00" * 20,
+                           expiration_enabled=True,
+                           expiration_mode=("date-cutoff",
+                                            then, ("mutable",)))
+        lc = ss.lease_checker
+        lc.slow_start = 0
+        webstatus = StorageStatus(ss)
+
+        self.make_shares(ss)
+        [immutable_si_0, immutable_si_1, mutable_si_2, mutable_si_3] = self.sis
+        # set all leases to be expirable
+        new_expiration_time = now - 3000 + 31*24*60*60
+
+        def count_shares(si):
+            return len(list(ss._iter_share_files(si)))
+        def _get_sharefile(si):
+            return list(ss._iter_share_files(si))[0]
+        def count_leases(si):
+            return len(list(_get_sharefile(si).get_leases()))
+
+        sf0 = _get_sharefile(immutable_si_0)
+        self.backdate_lease(sf0, self.renew_secrets[0], new_expiration_time)
+        sf1 = _get_sharefile(immutable_si_1)
+        self.backdate_lease(sf1, self.renew_secrets[1], new_expiration_time)
+        self.backdate_lease(sf1, self.renew_secrets[2], new_expiration_time)
+        sf2 = _get_sharefile(mutable_si_2)
+        self.backdate_lease(sf2, self.renew_secrets[3], new_expiration_time)
+        sf3 = _get_sharefile(mutable_si_3)
+        self.backdate_lease(sf3, self.renew_secrets[4], new_expiration_time)
+        self.backdate_lease(sf3, self.renew_secrets[5], new_expiration_time)
+
+        ss.setServiceParent(self.s)
+        def _wait():
+            return bool(lc.get_state()["last-cycle-finished"] is not None)
+        d = self.poll(_wait)
+
+        def _after_first_cycle(ignored):
+            self.failUnlessEqual(count_shares(immutable_si_0), 1)
+            self.failUnlessEqual(count_leases(immutable_si_0), 1)
+            self.failUnlessEqual(count_shares(immutable_si_1), 1)
+            self.failUnlessEqual(count_leases(immutable_si_1), 2)
+            self.failUnlessEqual(count_shares(mutable_si_2), 0)
+            self.failUnlessEqual(count_shares(mutable_si_3), 0)
+        d.addCallback(_after_first_cycle)
+        d.addCallback(lambda ign: self.render1(webstatus))
+        def _check_html(html):
+            s = remove_tags(html)
+            self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s)
+        d.addCallback(_check_html)
+        return d
+
     def test_bad_mode(self):
         basedir = "storage/LeaseCrawler/bad_mode"
         fileutil.make_dirs(basedir)

src/allmydata/web/storage.py

@@ -130,14 +130,18 @@ class StorageStatus(rend.Page):
     def render_lease_expiration_mode(self, ctx, data):
         mode = self.storage.lease_checker.mode
         if mode[0] == "age":
-            return ctx.tag["leases created or last renewed more than %s ago "
-                           "will be considered expired"
-                           % abbreviate_time(mode[1])]
+            ctx.tag["leases created or last renewed more than %s ago "
+                    "will be considered expired"
+                    % abbreviate_time(mode[1])]
         else:
             assert mode[0] == "date-cutoff"
             date = time.strftime("%d-%b-%Y", time.gmtime(mode[1]))
-            return ctx.tag["leases created or last renewed before %s "
-                           "will be considered expired" % date]
+            ctx.tag["leases created or last renewed before %s "
+                    "will be considered expired" % date]
+        if len(mode) > 2:
+            ctx.tag[", and only the following sharetypes will be expired: ",
+                    sorted(mode[2])]
+        return ctx.tag
 
     def format_recovered(self, sr, a):
         def maybe(d):