ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-20 17:52:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

expirer: change setup, config options, in preparation for adding tahoe.cfg controls

Browse Source
This commit is contained in:
Brian Warner 2009-03-18 17:21:38 -07:00
parent fffab0d724
commit 8eaee28550
4 changed files with 84 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/allmydata/storage/expirer.py
View File

@ -49,19 +49,25 @@ class LeaseCheckingCrawler(ShareCrawler):
minimum_cycle_time = 12*60*60 # not more than twice per day
def __init__(self, server, statefile, historyfile,
expiration_enabled, expiration_mode):
expiration_enabled, mode,
override_lease_duration, # used if expiration_mode=="age"
date_cutoff, # used if expiration_mode=="date-cutoff"
sharetypes):
self.historyfile = historyfile
self.expiration_enabled = expiration_enabled
self.mode = expiration_mode
if self.mode[0] not in ("age", "date-cutoff"):
raise ValueError("garbage-collection mode '%s' must be 'age' or 'date-cutoff'" % self.mode[0])
if self.mode[0] == "age":
assert isinstance(expiration_mode[1], int) # seconds
elif self.mode[0] == "date-cutoff":
assert isinstance(expiration_mode[1], int) # seconds-since-epoch
self.sharetypes_to_expire = ("mutable", "immutable")
if len(self.mode) > 2:
self.sharetypes_to_expire = self.mode[2]
self.mode = mode
self.override_lease_duration = None
self.date_cutoff = None
if self.mode == "age":
assert isinstance(override_lease_duration, (int, type(None)))
self.override_lease_duration = override_lease_duration # seconds
elif self.mode == "date-cutoff":
assert isinstance(date_cutoff, int) # seconds-since-epoch
assert date_cutoff is not None
self.date_cutoff = date_cutoff
else:
raise ValueError("GC mode '%s' must be 'age' or 'date-cutoff'" % mode)
self.sharetypes_to_expire = sharetypes
ShareCrawler.__init__(self, server, statefile)
def add_initial_state(self):
@ -179,14 +185,15 @@ class LeaseCheckingCrawler(ShareCrawler):
# expired-or-not according to our configured age limit
expired = False
if self.mode[0] == "age":
age_limit = self.mode[1]
if self.mode == "age":
age_limit = original_expiration_time
if self.override_lease_duration is not None:
age_limit = self.override_lease_duration
if age > age_limit:
expired = True
else:
assert self.mode[0] == "date-cutoff"
date_cutoff = self.mode[1]
if grant_renew_time < date_cutoff:
assert self.mode == "date-cutoff"
if grant_renew_time < self.date_cutoff:
expired = True
if sharetype not in self.sharetypes_to_expire:
expired = False
@ -269,7 +276,10 @@ class LeaseCheckingCrawler(ShareCrawler):
now = time.time()
h["cycle-start-finish-times"] = (start, now)
h["expiration-enabled"] = self.expiration_enabled
h["configured-expiration-mode"] = self.mode
h["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.date_cutoff,
self.sharetypes_to_expire)
s = self.state["cycle-to-date"]
@ -376,7 +386,10 @@ class LeaseCheckingCrawler(ShareCrawler):
lah = so_far["lease-age-histogram"]
so_far["lease-age-histogram"] = self.convert_lease_age_histogram(lah)
so_far["expiration-enabled"] = self.expiration_enabled
so_far["configured-expiration-mode"] = self.mode
so_far["configured-expiration-mode"] = (self.mode,
self.override_lease_duration,
self.date_cutoff,
self.sharetypes_to_expire)
so_far_sr = so_far["space-recovered"]
remaining_sr = {}

25
src/allmydata/storage/server.py
View File

@ -41,7 +41,10 @@ class StorageServer(service.MultiService, Referenceable):
discard_storage=False, readonly_storage=False,
stats_provider=None,
expiration_enabled=False,
expiration_mode=("age", 31*24*60*60)):
expiration_mode="age",
expiration_override_lease_duration=None,
expiration_date_cutoff=None,
expiration_sharetypes=("mutable", "immutable")):
service.MultiService.__init__(self)
assert isinstance(nodeid, str)
assert len(nodeid) == 20
@ -82,22 +85,22 @@ class StorageServer(service.MultiService, Referenceable):
"cancel": [],
}
self.add_bucket_counter()
self.add_lease_checker(expiration_enabled, expiration_mode)
statefile = os.path.join(self.storedir, "lease_checker.state")
historyfile = os.path.join(self.storedir, "lease_checker.history")
klass = self.LeaseCheckerClass
self.lease_checker = klass(self, statefile, historyfile,
expiration_enabled, expiration_mode,
expiration_override_lease_duration,
expiration_date_cutoff,
expiration_sharetypes)
self.lease_checker.setServiceParent(self)
def add_bucket_counter(self):
statefile = os.path.join(self.storedir, "bucket_counter.state")
self.bucket_counter = BucketCountingCrawler(self, statefile)
self.bucket_counter.setServiceParent(self)
def add_lease_checker(self, expiration_enabled, expiration_mode):
statefile = os.path.join(self.storedir, "lease_checker.state")
historyfile = os.path.join(self.storedir, "lease_checker.history")
klass = self.LeaseCheckerClass
self.lease_checker = klass(self, statefile, historyfile,
expiration_enabled=expiration_enabled,
expiration_mode=expiration_mode)
self.lease_checker.setServiceParent(self)
def count(self, name, delta=1):
if self.stats_provider:
self.stats_provider.count("storage_server." + name, delta)

40
src/allmydata/test/test_storage.py
View File

@ -1753,7 +1753,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
# than 2000s old will be expired.
ss = InstrumentedStorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode=("age",2000))
expiration_mode="age",
expiration_override_lease_duration=2000)
# make it start sooner than usual.
lc = ss.lease_checker
lc.slow_start = 0
@ -1845,7 +1846,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
last = s["history"][0]
self.failUnlessEqual(last["expiration-enabled"], True)
self.failUnlessEqual(last["configured-expiration-mode"], ("age",2000))
self.failUnlessEqual(last["configured-expiration-mode"],
("age", 2000, None, ("mutable", "immutable")))
self.failUnlessEqual(last["leases-per-share-histogram"], {1: 2, 2: 2})
rec = last["space-recovered"]
@ -1874,7 +1876,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
def _check_html(html):
s = remove_tags(html)
self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
self.failUnlessIn("leases created or last renewed more than 33 minutes ago will be considered expired", s)
self.failUnlessIn("Leases created or last renewed more than 33 minutes ago will be considered expired.", s)
self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
d.addCallback(_check_html)
return d
@ -1888,7 +1890,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
then = int(now - 2000)
ss = InstrumentedStorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode=("date-cutoff",then))
expiration_mode="date-cutoff",
expiration_date_cutoff=then)
# make it start sooner than usual.
lc = ss.lease_checker
lc.slow_start = 0
@ -1985,7 +1988,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
self.failUnlessEqual(last["expiration-enabled"], True)
self.failUnlessEqual(last["configured-expiration-mode"],
("date-cutoff",then))
("date-cutoff", None, then,
("mutable", "immutable")))
self.failUnlessEqual(last["leases-per-share-histogram"],
{1: 2, 2: 2})
@ -2017,8 +2021,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
self.failUnlessIn("Expiration Enabled:"
" expired leases will be removed", s)
date = time.strftime("%d-%b-%Y", time.gmtime(then))
self.failUnlessIn("leases created or last renewed before %s"
" will be considered expired" % date, s)
self.failUnlessIn("Leases created or last renewed before %s"
" will be considered expired." % date, s)
self.failUnlessIn(" recovered: 2 shares, 2 buckets (1 mutable / 1 immutable), ", s)
d.addCallback(_check_html)
return d
@ -2030,8 +2034,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode=("date-cutoff",
then, ("immutable",)))
expiration_mode="date-cutoff",
expiration_date_cutoff=then,
expiration_sharetypes=("immutable",))
lc = ss.lease_checker
lc.slow_start = 0
webstatus = StorageStatus(ss)
@ -2075,7 +2080,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
d.addCallback(lambda ign: self.render1(webstatus))
def _check_html(html):
s = remove_tags(html)
self.failUnlessIn("only the following sharetypes will be expired: immutable Next crawl", s)
self.failUnlessIn("The following sharetypes will be expired: immutable.", s)
d.addCallback(_check_html)
return d
@ -2086,8 +2091,9 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
then = int(now - 2000)
ss = StorageServer(basedir, "\x00" * 20,
expiration_enabled=True,
expiration_mode=("date-cutoff",
then, ("mutable",)))
expiration_mode="date-cutoff",
expiration_date_cutoff=then,
expiration_sharetypes=("mutable",))
lc = ss.lease_checker
lc.slow_start = 0
webstatus = StorageStatus(ss)
@ -2131,7 +2137,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
d.addCallback(lambda ign: self.render1(webstatus))
def _check_html(html):
s = remove_tags(html)
self.failUnlessIn("only the following sharetypes will be expired: mutable Next crawl", s)
self.failUnlessIn("The following sharetypes will be expired: mutable.", s)
d.addCallback(_check_html)
return d
@ -2140,9 +2146,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
fileutil.make_dirs(basedir)
e = self.failUnlessRaises(ValueError,
StorageServer, basedir, "\x00" * 20,
expiration_mode=("bogus", 0))
self.failUnless("garbage-collection mode 'bogus'"
" must be 'age' or 'date-cutoff'" in str(e), str(e))
expiration_mode="bogus")
self.failUnless("GC mode 'bogus' must be 'age' or 'date-cutoff'" in str(e), str(e))
def test_parse_duration(self):
DAY = 24*60*60
@ -2254,7 +2259,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
basedir = "storage/LeaseCrawler/no_st_blocks"
fileutil.make_dirs(basedir)
ss = No_ST_BLOCKS_StorageServer(basedir, "\x00" * 20,
expiration_mode=("age",-1000))
expiration_mode="age",
expiration_override_lease_duration=-1000)
# a negative expiration_time= means the "configured-"
# space-recovered counts will be non-zero, since all shares will have
# expired by then

28
src/allmydata/web/storage.py
View File

@ -128,19 +128,23 @@ class StorageStatus(rend.Page):
return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
def render_lease_expiration_mode(self, ctx, data):
mode = self.storage.lease_checker.mode
if mode[0] == "age":
ctx.tag["leases created or last renewed more than %s ago "
"will be considered expired"
% abbreviate_time(mode[1])]
lc = self.storage.lease_checker
if lc.mode == "age":
if lc.override_lease_duration is None:
ctx.tag["Leases will expire naturally, probably 31 days after "
"creation or renewal."]
else:
ctx.tag["Leases created or last renewed more than %s ago "
"will be considered expired."
% abbreviate_time(lc.override_lease_duration)]
else:
assert mode[0] == "date-cutoff"
date = time.strftime("%d-%b-%Y", time.gmtime(mode[1]))
ctx.tag["leases created or last renewed before %s "
"will be considered expired" % date]
if len(mode) > 2:
ctx.tag[", and only the following sharetypes will be expired: ",
sorted(mode[2])]
assert lc.mode == "date-cutoff"
date = time.strftime("%d-%b-%Y", time.gmtime(lc.date_cutoff))
ctx.tag["Leases created or last renewed before %s "
"will be considered expired." % date]
if len(lc.mode) > 2:
ctx.tag[" The following sharetypes will be expired: ",
sorted(lc.sharetypes_to_expire), "."]
return ctx.tag
def format_recovered(self, sr, a):