expirer: clean up constructor args, add tahoe.cfg controls, use cutoff_date instead of date_cutoff

docs/proposed/garbage-collection.txt

@@ -114,11 +114,11 @@ expire.enabled = (boolean, optional)
  have expired. Other controls dictate when leases are considered to have
  expired. The default is False.
 
-expire.mode = (string, "age" or "date-cutoff", required if expiration enabled)
+expire.mode = (string, "age" or "cutoff-date", required if expiration enabled)
 
  If this string is "age", the age-based expiration scheme is used, and the
  "expire.override_lease_duration" setting can be provided to influence the
- lease ages. If it is "date-cutoff", the absolute-date-cutoff mode is used,
+ lease ages. If it is "cutoff-date", the absolute-date-cutoff mode is used,
  and the "expire.cutoff_date" setting must be provided to specify the cutoff
  date. The mode setting currently has no default: you must provide a value.
 
@@ -160,12 +160,12 @@ expire.override_lease_duration = (duration string, optional)
  been passed.
 
  This key is only valid when age-based expiration is in use (i.e. when
- "expire.mode = age" is used). It will be rejected if date-cutoff expiration
+ "expire.mode = age" is used). It will be rejected if cutoff-date expiration
  is in use.
 
-expire.cutoff_date = (date string, required if mode=date-cutoff)
+expire.cutoff_date = (date string, required if mode=cutoff-date)
 
- When date-cutoff expiration is in use, a lease will be expired if its
+ When cutoff-date expiration is in use, a lease will be expired if its
  create/renew timestamp is older than the cutoff date. This string will be a
  date in the following format:
 
@@ -179,7 +179,7 @@ expire.cutoff_date = (date string, required if mode=date-cutoff)
  last renewal time and the cutoff date.
 
  This key is only valid when cutoff-based expiration is in use (i.e. when
- "expire.mode = date-cutoff"). It will be rejected if age-based expiration is
+ "expire.mode = cutoff-date"). It will be rejected if age-based expiration is
  in use.
 
 expire.immutable = (boolean, optional)

src/allmydata/client.py

@@ -19,6 +19,7 @@ from allmydata.control import ControlServer
 from allmydata.introducer.client import IntroducerClient
 from allmydata.util import hashutil, base32, pollmixin, cachedir
 from allmydata.util.abbreviate import parse_abbreviated_size
+from allmydata.util.time_format import parse_duration, parse_date
 from allmydata.uri import LiteralFileURI
 from allmydata.dirnode import NewDirectoryNode
 from allmydata.mutable.filenode import MutableFileNode
@@ -164,12 +165,41 @@ class Client(node.Node, pollmixin.PollMixin):
             reserved = 0
         discard = self.get_config("storage", "debug_discard", False,
                                   boolean=True)
+
+        expire = self.get_config("storage", "expire.enabled", False, boolean=True)
+        if expire:
+            mode = self.get_config("storage", "expire.mode") # require a mode
+        else:
+            mode = self.get_config("storage", "expire.mode", "age")
+
+        o_l_d = self.get_config("storage", "expire.override_lease_duration", None)
+        if o_l_d is not None:
+            o_l_d = parse_duration(o_l_d)
+
+        cutoff_date = None
+        if mode == "cutoff-date":
+            cutoff_date = self.get_config("storage", "expire.cutoff_date")
+            cutoff_date = parse_date(cutoff_date)
+
+        sharetypes = []
+        if self.get_config("storage", "expire.immutable", True, boolean=True):
+            sharetypes.append("immutable")
+        if self.get_config("storage", "expire.mutable", True, boolean=True):
+            sharetypes.append("mutable")
+        expiration_sharetypes = tuple(sharetypes)
+
         ss = StorageServer(storedir, self.nodeid,
                            reserved_space=reserved,
                            discard_storage=discard,
                            readonly_storage=readonly,
-                           stats_provider=self.stats_provider)
+                           stats_provider=self.stats_provider,
+                           expiration_enabled=expire,
+                           expiration_mode=mode,
+                           expiration_override_lease_duration=o_l_d,
+                           expiration_cutoff_date=cutoff_date,
+                           expiration_sharetypes=expiration_sharetypes)
         self.add_service(ss)
+
         d = self.when_tub_ready()
         # we can't do registerReference until the Tub is ready
         def _publish(res):

src/allmydata/storage/expirer.py

@@ -51,22 +51,22 @@ class LeaseCheckingCrawler(ShareCrawler):
     def __init__(self, server, statefile, historyfile,
                  expiration_enabled, mode,
                  override_lease_duration, # used if expiration_mode=="age"
-                 date_cutoff, # used if expiration_mode=="date-cutoff"
+                 cutoff_date, # used if expiration_mode=="cutoff-date"
                  sharetypes):
         self.historyfile = historyfile
         self.expiration_enabled = expiration_enabled
         self.mode = mode
         self.override_lease_duration = None
-        self.date_cutoff = None
+        self.cutoff_date = None
         if self.mode == "age":
             assert isinstance(override_lease_duration, (int, type(None)))
             self.override_lease_duration = override_lease_duration # seconds
-        elif self.mode == "date-cutoff":
+        elif self.mode == "cutoff-date":
-            assert isinstance(date_cutoff, int) # seconds-since-epoch
+            assert isinstance(cutoff_date, int) # seconds-since-epoch
-            assert date_cutoff is not None
+            assert cutoff_date is not None
-            self.date_cutoff = date_cutoff
+            self.cutoff_date = cutoff_date
         else:
-            raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode)
+            raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode)
         self.sharetypes_to_expire = sharetypes
         ShareCrawler.__init__(self, server, statefile)
 
@@ -192,8 +192,8 @@ class LeaseCheckingCrawler(ShareCrawler):
                 if age > age_limit:
                     expired = True
             else:
-                assert self.mode == "date-cutoff"
+                assert self.mode == "cutoff-date"
-                if grant_renew_time < self.date_cutoff:
+                if grant_renew_time < self.cutoff_date:
                     expired = True
             if sharetype not in self.sharetypes_to_expire:
                 expired = False
@@ -278,7 +278,7 @@ class LeaseCheckingCrawler(ShareCrawler):
         h["expiration-enabled"] = self.expiration_enabled
         h["configured-expiration-mode"] = (self.mode,
                                            self.override_lease_duration,
-                                           self.date_cutoff,
+                                           self.cutoff_date,
                                            self.sharetypes_to_expire)
 
         s = self.state["cycle-to-date"]
@@ -388,7 +388,7 @@ class LeaseCheckingCrawler(ShareCrawler):
         so_far["expiration-enabled"] = self.expiration_enabled
         so_far["configured-expiration-mode"] = (self.mode,
                                                 self.override_lease_duration,
-                                                self.date_cutoff,
+                                                self.cutoff_date,
                                                 self.sharetypes_to_expire)
 
         so_far_sr = so_far["space-recovered"]

src/allmydata/storage/server.py

@@ -43,7 +43,7 @@ class StorageServer(service.MultiService, Referenceable):
                  expiration_enabled=False,
                  expiration_mode="age",
                  expiration_override_lease_duration=None,
-                 expiration_date_cutoff=None,
+                 expiration_cutoff_date=None,
                  expiration_sharetypes=("mutable", "immutable")):
         service.MultiService.__init__(self)
         assert isinstance(nodeid, str)
@@ -92,7 +92,7 @@ class StorageServer(service.MultiService, Referenceable):
         self.lease_checker = klass(self, statefile, historyfile,
                                    expiration_enabled, expiration_mode,
                                    expiration_override_lease_duration,
-                                   expiration_date_cutoff,
+                                   expiration_cutoff_date,
                                    expiration_sharetypes)
         self.lease_checker.setServiceParent(self)
 

src/allmydata/test/test_storage.py

@@ -1881,17 +1881,17 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         d.addCallback(_check_html)
         return d
 
-    def test_expire_date_cutoff(self):
+    def test_expire_cutoff_date(self):
-        basedir = "storage/LeaseCrawler/expire_date_cutoff"
+        basedir = "storage/LeaseCrawler/expire_cutoff_date"
         fileutil.make_dirs(basedir)
-        # setting date-cutoff to 2000 seconds ago means that any lease which
+        # setting cutoff-date to 2000 seconds ago means that any lease which
         # is more than 2000s old will be expired.
         now = time.time()
         then = int(now - 2000)
         ss = InstrumentedStorageServer(basedir, "\x00" * 20,
                                        expiration_enabled=True,
-                                       expiration_mode="date-cutoff",
+                                       expiration_mode="cutoff-date",
-                                       expiration_date_cutoff=then)
+                                       expiration_cutoff_date=then)
         # make it start sooner than usual.
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -1988,7 +1988,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
 
             self.failUnlessEqual(last["expiration-enabled"], True)
             self.failUnlessEqual(last["configured-expiration-mode"],
-                                 ("date-cutoff", None, then,
+                                 ("cutoff-date", None, then,
                                   ("mutable", "immutable")))
             self.failUnlessEqual(last["leases-per-share-histogram"],
                                  {1: 2, 2: 2})
@@ -2034,8 +2034,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         then = int(now - 2000)
         ss = StorageServer(basedir, "\x00" * 20,
                            expiration_enabled=True,
-                           expiration_mode="date-cutoff",
+                           expiration_mode="cutoff-date",
-                           expiration_date_cutoff=then,
+                           expiration_cutoff_date=then,
                            expiration_sharetypes=("immutable",))
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -2091,8 +2091,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         then = int(now - 2000)
         ss = StorageServer(basedir, "\x00" * 20,
                            expiration_enabled=True,
-                           expiration_mode="date-cutoff",
+                           expiration_mode="cutoff-date",
-                           expiration_date_cutoff=then,
+                           expiration_cutoff_date=then,
                            expiration_sharetypes=("mutable",))
         lc = ss.lease_checker
         lc.slow_start = 0
@@ -2147,7 +2147,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
         e = self.failUnlessRaises(ValueError,
                                   StorageServer, basedir, "\x00" * 20,
                                   expiration_mode="bogus")
-        self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e))
+        self.failUnless("GC mode 'bogus' must be 'age' or 'cutoff-date'" in str(e), str(e))
 
     def test_parse_duration(self):
         DAY = 24*60*60