ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-07 10:56:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

expirer: clean up constructor args, add tahoe.cfg controls, use cutoff_date instead of date_cutoff

Browse Source
This commit is contained in:
Brian Warner 2009-03-18 18:00:09 -07:00
parent babcf632da
commit f0071c2571
5 changed files with 61 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/proposed/garbage-collection.txt
View File

@ -114,11 +114,11 @@ expire.enabled = (boolean, optional)
have expired. Other controls dictate when leases are considered to have
expired. The default is False.
expire.mode = (string, "age" or "date-cutoff", required if expiration enabled)
expire.mode = (string, "age" or "cutoff-date", required if expiration enabled)
If this string is "age", the age-based expiration scheme is used, and the
"expire.override_lease_duration" setting can be provided to influence the
lease ages. If it is "date-cutoff", the absolute-date-cutoff mode is used,
lease ages. If it is "cutoff-date", the absolute-date-cutoff mode is used,
and the "expire.cutoff_date" setting must be provided to specify the cutoff
date. The mode setting currently has no default: you must provide a value.
@ -160,12 +160,12 @@ expire.override_lease_duration = (duration string, optional)
been passed.
This key is only valid when age-based expiration is in use (i.e. when
"expire.mode = age" is used). It will be rejected if date-cutoff expiration
"expire.mode = age" is used). It will be rejected if cutoff-date expiration
is in use.
expire.cutoff_date = (date string, required if mode=date-cutoff)
expire.cutoff_date = (date string, required if mode=cutoff-date)
When date-cutoff expiration is in use, a lease will be expired if its
When cutoff-date expiration is in use, a lease will be expired if its
create/renew timestamp is older than the cutoff date. This string will be a
date in the following format:
@ -179,7 +179,7 @@ expire.cutoff_date = (date string, required if mode=date-cutoff)
last renewal time and the cutoff date.
This key is only valid when cutoff-based expiration is in use (i.e. when
"expire.mode = date-cutoff"). It will be rejected if age-based expiration is
"expire.mode = cutoff-date"). It will be rejected if age-based expiration is
in use.
expire.immutable = (boolean, optional)

32
src/allmydata/client.py
View File

@ -19,6 +19,7 @@ from allmydata.control import ControlServer
from allmydata.introducer.client import IntroducerClient
from allmydata.util import hashutil, base32, pollmixin, cachedir
from allmydata.util.abbreviate import parse_abbreviated_size
from allmydata.util.time_format import parse_duration, parse_date
from allmydata.uri import LiteralFileURI
from allmydata.dirnode import NewDirectoryNode
from allmydata.mutable.filenode import MutableFileNode
@ -164,12 +165,41 @@ class Client(node.Node, pollmixin.PollMixin):
reserved = 0
discard = self.get_config("storage", "debug_discard", False,
boolean=True)
expire = self.get_config("storage", "expire.enabled", False, boolean=True)
if expire:
mode = self.get_config("storage", "expire.mode") # require a mode
else:
mode = self.get_config("storage", "expire.mode", "age")
o_l_d = self.get_config("storage", "expire.override_lease_duration", None)
if o_l_d is not None:
o_l_d = parse_duration(o_l_d)
cutoff_date = None
if mode == "cutoff-date":
cutoff_date = self.get_config("storage", "expire.cutoff_date")
cutoff_date = parse_date(cutoff_date)
sharetypes = []
if self.get_config("storage", "expire.immutable", True, boolean=True):
sharetypes.append("immutable")
if self.get_config("storage", "expire.mutable", True, boolean=True):
sharetypes.append("mutable")
expiration_sharetypes = tuple(sharetypes)
ss = StorageServer(storedir, self.nodeid,
reserved_space=reserved,
discard_storage=discard,
readonly_storage=readonly,
stats_provider=self.stats_provider)
stats_provider=self.stats_provider,
expiration_enabled=expire,
expiration_mode=mode,
expiration_override_lease_duration=o_l_d,
expiration_cutoff_date=cutoff_date,
expiration_sharetypes=expiration_sharetypes)
self.add_service(ss)
d = self.when_tub_ready()
# we can't do registerReference until the Tub is ready
def _publish(res):

22
src/allmydata/storage/expirer.py
View File

@ -51,22 +51,22 @@ class LeaseCheckingCrawler(ShareCrawler):
def __init__(self, server, statefile, historyfile,
expiration_enabled, mode,
override_lease_duration, # used if expiration_mode=="age"
date_cutoff, # used if expiration_mode=="date-cutoff"
cutoff_date, # used if expiration_mode=="cutoff-date"
sharetypes):
self.historyfile = historyfile
self.expiration_enabled = expiration_enabled
self.mode = mode
self.override_lease_duration = None
self.date_cutoff = None
self.cutoff_date = None
if self.mode == "age":
assert isinstance(override_lease_duration, (int, type(None)))
self.override_lease_duration = override_lease_duration # seconds
elif self.mode == "date-cutoff":
assert isinstance(date_cutoff, int) # seconds-since-epoch
assert date_cutoff is not None
self.date_cutoff = date_cutoff
elif self.mode == "cutoff-date":
assert isinstance(cutoff_date, int) # seconds-since-epoch
assert cutoff_date is not None
self.cutoff_date = cutoff_date
else:
raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode)
raise ValueError("GC mode '%s' must be 'age' or 'cutoff-date'" % mode)
self.sharetypes_to_expire = sharetypes
ShareCrawler.__init__(self, server, statefile)
@ -192,8 +192,8 @@ class LeaseCheckingCrawler(ShareCrawler):
if age > age_limit:
expired = True
else:
assert self.mode == "date-cutoff"
if grant_renew_time < self.date_cutoff:
assert self.mode == "cutoff-date"
if grant_renew_time < self.cutoff_date:
expired = True
if sharetype not in self.sharetypes_to_expire:
expired = False
@ -278,7 +278,7 @@ class LeaseCheckingCrawler(ShareCrawler):
h["expiration-enabled"] = self.expiration_enabled
h["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.date_cutoff,
self.cutoff_date,
self.sharetypes_to_expire)
s = self.state["cycle-to-date"]
@ -388,7 +388,7 @@ class LeaseCheckingCrawler(ShareCrawler):
so_far["expiration-enabled"] = self.expiration_enabled
so_far["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.date_cutoff,
self.cutoff_date,
self.sharetypes_to_expire)
so_far_sr = so_far["space-recovered"]

4
src/allmydata/storage/server.py
View File

@ -43,7 +43,7 @@ class StorageServer(service.MultiService, Referenceable):
expiration_enabled=False,
expiration_mode="age",
expiration_override_lease_duration=None,
expiration_date_cutoff=None,
expiration_cutoff_date=None,
expiration_sharetypes=("mutable", "immutable")):
service.MultiService.__init__(self)
assert isinstance(nodeid, str)
@ -92,7 +92,7 @@ class StorageServer(service.MultiService, Referenceable):
self.lease_checker = klass(self, statefile, historyfile,
expiration_enabled, expiration_mode,
expiration_override_lease_duration,
expiration_date_cutoff,
expiration_cutoff_date,
expiration_sharetypes)
self.lease_checker.setServiceParent(self)

22
src/allmydata/test/test_storage.py
View File

@ -1881,17 +1881,17 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
d.addCallback(_check_html)
return d
def test_expire_date_cutoff(self):
basedir = "storage/LeaseCrawler/expire_date_cutoff"
def test_expire_cutoff_date(self):
basedir = "storage/LeaseCrawler/expire_cutoff_date"
fileutil.make_dirs(basedir)
# setting date-cutoff to 2000 seconds ago means that any lease which
# setting cutoff-date to 2000 seconds ago means that any lease which
# is more than 2000s old will be expired.
now = time.time()
then = int(now - 2000)
ss = InstrumentedStorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode="date-cutoff",
expiration_date_cutoff=then)
expiration_mode="cutoff-date",
expiration_cutoff_date=then)
# make it start sooner than usual.
lc = ss.lease_checker
lc.slow_start = 0
@ -1988,7 +1988,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
self.failUnlessEqual(last["expiration-enabled"], True)
self.failUnlessEqual(last["configured-expiration-mode"],
("date-cutoff", None, then,
("cutoff-date", None, then,
("mutable", "immutable")))
self.failUnlessEqual(last["leases-per-share-histogram"],
{1: 2, 2: 2})
@ -2034,8 +2034,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode="date-cutoff",
expiration_date_cutoff=then,
expiration_mode="cutoff-date",
expiration_cutoff_date=then,
expiration_sharetypes=("immutable",))
lc = ss.lease_checker
lc.slow_start = 0
@ -2091,8 +2091,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode="date-cutoff",
expiration_date_cutoff=then,
expiration_mode="cutoff-date",
expiration_cutoff_date=then,
expiration_sharetypes=("mutable",))
lc = ss.lease_checker
lc.slow_start = 0
@ -2147,7 +2147,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
e = self.failUnlessRaises(ValueError,
StorageServer, basedir, "\x00" * 20,
expiration_mode="bogus")
self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e))
self.failUnless("GC mode 'bogus' must be 'age' or 'cutoff-date'" in str(e), str(e))
def test_parse_duration(self):
DAY = 24*60*60