mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-04-07 10:56:49 +00:00
Merge pull request #690 from sajith/3247.storage-py-neovw-to-twt
Use twisted.web.template in web/storage.py Fixes: ticket:3247
This commit is contained in:
Sajith Sasidharan
GitHub
commit
61aef4caef
0
newsfragments/3247.minor
Normal file
0
newsfragments/3247.minor
Normal file
@ -12,6 +12,17 @@ from twisted.trial import unittest
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.application import service
|
||||
from twisted.web.template import flattenString
|
||||
|
||||
# We need to use `nevow.inevow.IRequest` for now for compatibility
|
||||
# with the code in web/common.py. Once nevow bits are gone from
|
||||
# web/common.py, we can use `twisted.web.iweb.IRequest` here.
|
||||
from nevow.inevow import IRequest
|
||||
|
||||
from twisted.web.server import Request
|
||||
from twisted.web.test.requesthelper import DummyChannel
|
||||
from zope.interface import implementer
|
||||
|
||||
from foolscap.api import fireEventually
|
||||
import itertools
|
||||
from allmydata import interfaces
|
||||
@ -36,9 +47,12 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \
|
||||
SHARE_HASH_CHAIN_SIZE
|
||||
from allmydata.interfaces import BadWriteEnablerError
|
||||
from allmydata.test.common import LoggingServiceParent, ShouldFailMixin
|
||||
from allmydata.test.common_web import WebRenderingMixin
|
||||
from allmydata.test.no_network import NoNetworkServer
|
||||
from allmydata.web.storage import StorageStatus, remove_prefix
|
||||
from allmydata.web.storage import (
|
||||
StorageStatus,
|
||||
StorageStatusElement,
|
||||
remove_prefix
|
||||
)
|
||||
from allmydata.storage_client import (
|
||||
_StorageServer,
|
||||
)
|
||||
@ -2972,6 +2986,39 @@ def remove_tags(s):
|
||||
s = re.sub(r'\s+', ' ', s)
|
||||
return s
|
||||
|
||||
def renderSynchronously(ss):
|
||||
"""
|
||||
Return fully rendered HTML document.
|
||||
|
||||
:param _StorageStatus ss: a StorageStatus instance.
|
||||
"""
|
||||
return unittest.TestCase().successResultOf(renderDeferred(ss))
|
||||
|
||||
def renderDeferred(ss):
|
||||
"""
|
||||
Return a `Deferred` HTML renderer.
|
||||
|
||||
:param _StorageStatus ss: a StorageStatus instance.
|
||||
"""
|
||||
elem = StorageStatusElement(ss._storage, ss._nickname)
|
||||
return flattenString(None, elem)
|
||||
|
||||
def renderJSON(resource):
|
||||
"""Render a JSON from the given resource."""
|
||||
|
||||
@implementer(IRequest)
|
||||
class JSONRequest(Request):
|
||||
"""
|
||||
A Request with t=json argument added to it. This is useful to
|
||||
invoke a Resouce.render_JSON() method.
|
||||
"""
|
||||
def __init__(self):
|
||||
Request.__init__(self, DummyChannel())
|
||||
self.args = {"t": ["json"]}
|
||||
self.fields = {}
|
||||
|
||||
return resource.render(JSONRequest())
|
||||
|
||||
class MyBucketCountingCrawler(BucketCountingCrawler):
|
||||
def finished_prefix(self, cycle, prefix):
|
||||
BucketCountingCrawler.finished_prefix(self, cycle, prefix)
|
||||
@ -3008,7 +3055,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
w = StorageStatus(ss)
|
||||
|
||||
# this sample is before the crawler has started doing anything
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
@ -3031,7 +3078,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
self.failUnlessEqual(state["last-complete-prefix"],
|
||||
ss.bucket_counter.prefixes[0])
|
||||
ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn(" Current crawl ", s)
|
||||
self.failUnlessIn(" (next work in ", s)
|
||||
@ -3043,7 +3090,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
d.addCallback(lambda ignored: self.poll(_watch))
|
||||
def _check2(ignored):
|
||||
ss.bucket_counter.cpu_slice = orig_cpu_slice
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Total buckets: 0 (the number of", s)
|
||||
self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s)
|
||||
@ -3105,20 +3152,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
def _check_1(ignored):
|
||||
# no ETA is available yet
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (next work", s)
|
||||
|
||||
def _check_2(ignored):
|
||||
# one prefix has finished, so an ETA based upon that elapsed time
|
||||
# should be available.
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (ETA ", s)
|
||||
|
||||
def _check_3(ignored):
|
||||
# two prefixes have finished
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("complete (ETA ", s)
|
||||
d.callback("done")
|
||||
@ -3161,7 +3208,7 @@ class InstrumentedStorageServer(StorageServer):
|
||||
class No_ST_BLOCKS_StorageServer(StorageServer):
|
||||
LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler
|
||||
|
||||
class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
def setUp(self):
|
||||
self.s = service.MultiService()
|
||||
@ -3291,7 +3338,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failIfEqual(sr2["configured-diskbytes"], None)
|
||||
self.failIfEqual(sr2["original-sharebytes"], None)
|
||||
d.addCallback(_after_first_bucket)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html_in_cycle(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("So far, this cycle has examined "
|
||||
@ -3366,7 +3413,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessEqual(count_leases(mutable_si_2), 1)
|
||||
self.failUnlessEqual(count_leases(mutable_si_3), 2)
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("recovered: 0 shares, 0 buckets "
|
||||
@ -3375,7 +3422,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
"(2 mutable / 2 immutable),", s)
|
||||
self.failUnlessIn("but expiration was not enabled", s)
|
||||
d.addCallback(_check_html)
|
||||
d.addCallback(lambda ign: self.render_json(webstatus))
|
||||
d.addCallback(lambda ign: renderJSON(webstatus))
|
||||
def _check_json(raw):
|
||||
data = json.loads(raw)
|
||||
self.failUnlessIn("lease-checker", data)
|
||||
@ -3466,7 +3513,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
d2.addCallback(_after_first_bucket)
|
||||
return d2
|
||||
d.addCallback(_after_first_bucket)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html_in_cycle(html):
|
||||
s = remove_tags(html)
|
||||
# the first bucket encountered gets deleted, and its prefix
|
||||
@ -3525,7 +3572,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnless(rec["configured-diskbytes"] >= 0,
|
||||
rec["configured-diskbytes"])
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Expiration Enabled: expired leases will be removed", s)
|
||||
@ -3610,7 +3657,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
d2.addCallback(_after_first_bucket)
|
||||
return d2
|
||||
d.addCallback(_after_first_bucket)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html_in_cycle(html):
|
||||
s = remove_tags(html)
|
||||
# the first bucket encountered gets deleted, and its prefix
|
||||
@ -3671,7 +3718,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnless(rec["configured-diskbytes"] >= 0,
|
||||
rec["configured-diskbytes"])
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Expiration Enabled:"
|
||||
@ -3733,7 +3780,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessEqual(count_shares(mutable_si_3), 1)
|
||||
self.failUnlessEqual(count_leases(mutable_si_3), 2)
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("The following sharetypes will be expired: immutable.", s)
|
||||
@ -3790,7 +3837,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessEqual(count_shares(mutable_si_2), 0)
|
||||
self.failUnlessEqual(count_shares(mutable_si_3), 0)
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render1(webstatus))
|
||||
d.addCallback(lambda ign: renderDeferred(webstatus))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("The following sharetypes will be expired: mutable.", s)
|
||||
@ -4012,7 +4059,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)])
|
||||
d.addCallback(_after_first_bucket)
|
||||
|
||||
d.addCallback(lambda ign: self.render_json(w))
|
||||
d.addCallback(lambda ign: renderJSON(w))
|
||||
def _check_json(raw):
|
||||
data = json.loads(raw)
|
||||
# grr. json turns all dict keys into strings.
|
||||
@ -4021,7 +4068,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
# it also turns all tuples into lists
|
||||
self.failUnlessEqual(corrupt_shares, [[first_b32, 0]])
|
||||
d.addCallback(_check_json)
|
||||
d.addCallback(lambda ign: self.render1(w))
|
||||
d.addCallback(lambda ign: renderDeferred(w))
|
||||
def _check_html(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s)
|
||||
@ -4039,14 +4086,14 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessEqual(rec["examined-shares"], 3)
|
||||
self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)])
|
||||
d.addCallback(_after_first_cycle)
|
||||
d.addCallback(lambda ign: self.render_json(w))
|
||||
d.addCallback(lambda ign: renderJSON(w))
|
||||
def _check_json_history(raw):
|
||||
data = json.loads(raw)
|
||||
last = data["lease-checker"]["history"]["0"]
|
||||
corrupt_shares = last["corrupt-shares"]
|
||||
self.failUnlessEqual(corrupt_shares, [[first_b32, 0]])
|
||||
d.addCallback(_check_json_history)
|
||||
d.addCallback(lambda ign: self.render1(w))
|
||||
d.addCallback(lambda ign: renderDeferred(w))
|
||||
def _check_html_history(html):
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s)
|
||||
@ -4059,11 +4106,8 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
d.addBoth(_cleanup)
|
||||
return d
|
||||
|
||||
def render_json(self, page):
|
||||
d = self.render1(page, args={"t": ["json"]})
|
||||
return d
|
||||
|
||||
class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
class WebStatus(unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
def setUp(self):
|
||||
self.s = service.MultiService()
|
||||
@ -4073,7 +4117,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
|
||||
def test_no_server(self):
|
||||
w = StorageStatus(None)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>No Storage Server Running</h1>", html)
|
||||
|
||||
def test_status(self):
|
||||
@ -4083,7 +4127,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, nodeid)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss, "nickname")
|
||||
d = self.render1(w)
|
||||
d = renderDeferred(w)
|
||||
def _check_html(html):
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
@ -4092,7 +4136,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
self.failUnlessIn("Reserved space: - 0 B (0)", s)
|
||||
d.addCallback(_check_html)
|
||||
d.addCallback(lambda ign: self.render_json(w))
|
||||
d.addCallback(lambda ign: renderJSON(w))
|
||||
def _check_json(raw):
|
||||
data = json.loads(raw)
|
||||
s = data["stats"]
|
||||
@ -4103,9 +4147,6 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
d.addCallback(_check_json)
|
||||
return d
|
||||
|
||||
def render_json(self, page):
|
||||
d = self.render1(page, args={"t": ["json"]})
|
||||
return d
|
||||
|
||||
def test_status_no_disk_stats(self):
|
||||
def call_get_disk_stats(whichdir, reserved_space=0):
|
||||
@ -4119,7 +4160,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, "\x00" * 20)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: Yes", s)
|
||||
@ -4139,7 +4180,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, "\x00" * 20)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: No", s)
|
||||
@ -4175,7 +4216,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
@ -4193,7 +4234,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Accepting new shares: No", s)
|
||||
@ -4204,7 +4245,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
|
||||
@ -4215,16 +4256,16 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin):
|
||||
ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6)
|
||||
ss.setServiceParent(self.s)
|
||||
w = StorageStatus(ss)
|
||||
html = w.renderSynchronously()
|
||||
html = renderSynchronously(w)
|
||||
self.failUnlessIn("<h1>Storage Server Status</h1>", html)
|
||||
s = remove_tags(html)
|
||||
self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s)
|
||||
|
||||
def test_util(self):
|
||||
w = StorageStatus(None)
|
||||
self.failUnlessEqual(w.render_space(None, None), "?")
|
||||
self.failUnlessEqual(w.render_space(None, 10e6), "10000000")
|
||||
self.failUnlessEqual(w.render_abbrev_space(None, None), "?")
|
||||
self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB")
|
||||
w = StorageStatusElement(None, None)
|
||||
self.failUnlessEqual(w.render_space(None), "?")
|
||||
self.failUnlessEqual(w.render_space(10e6), "10000000")
|
||||
self.failUnlessEqual(w.render_abbrev_space(None), "?")
|
||||
self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB")
|
||||
self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar")
|
||||
self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None)
|
||||
|
||||
@ -954,8 +954,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
|
||||
def test_storage(self):
|
||||
d = self.GET("/storage")
|
||||
def _check(res):
|
||||
self.failUnlessIn('Storage Server Status', res)
|
||||
self.failUnlessIn(FAVICON_MARKUP, res)
|
||||
soup = BeautifulSoup(res, 'html5lib')
|
||||
assert_soup_has_text(self, soup, 'Storage Server Status')
|
||||
assert_soup_has_favicon(self, soup)
|
||||
res_u = res.decode('utf-8')
|
||||
self.failUnlessIn(u'<li>Server Nickname: <span class="nickname mine">fake_nickname \u263A</span></li>', res_u)
|
||||
d.addCallback(_check)
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
|
||||
import time, json
|
||||
from nevow import rend, tags as T
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.web.template import (
|
||||
Element,
|
||||
XMLFile,
|
||||
tags as T,
|
||||
renderer,
|
||||
renderElement
|
||||
)
|
||||
from allmydata.web.common import (
|
||||
getxmlfile,
|
||||
abbreviate_time,
|
||||
MultiFormatPage,
|
||||
MultiFormatResource
|
||||
)
|
||||
from allmydata.util.abbreviate import abbreviate_space
|
||||
from allmydata.util import time_format, idlib
|
||||
@ -16,91 +22,108 @@ def remove_prefix(s, prefix):
|
||||
return s[len(prefix):]
|
||||
|
||||
|
||||
class StorageStatus(MultiFormatPage):
|
||||
docFactory = getxmlfile("storage_status.xhtml")
|
||||
# the default 'data' argument is the StorageServer instance
|
||||
class StorageStatusElement(Element):
|
||||
"""Class to render a storage status page."""
|
||||
|
||||
loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml"))
|
||||
|
||||
def __init__(self, storage, nickname=""):
|
||||
rend.Page.__init__(self, storage)
|
||||
self.storage = storage
|
||||
self.nickname = nickname
|
||||
"""
|
||||
:param _StorageServer storage: data about storage.
|
||||
:param string nickname: friendly name for storage.
|
||||
"""
|
||||
super(StorageStatusElement, self).__init__()
|
||||
self._storage = storage
|
||||
self._nickname = nickname
|
||||
|
||||
def render_JSON(self, req):
|
||||
req.setHeader("content-type", "text/plain")
|
||||
d = {"stats": self.storage.get_stats(),
|
||||
"bucket-counter": self.storage.bucket_counter.get_state(),
|
||||
"lease-checker": self.storage.lease_checker.get_state(),
|
||||
"lease-checker-progress": self.storage.lease_checker.get_progress(),
|
||||
}
|
||||
return json.dumps(d, indent=1) + "\n"
|
||||
@renderer
|
||||
def nickname(self, req, tag):
|
||||
return tag(self._nickname)
|
||||
|
||||
def data_nickname(self, ctx, storage):
|
||||
return self.nickname
|
||||
def data_nodeid(self, ctx, storage):
|
||||
return idlib.nodeid_b2a(self.storage.my_nodeid)
|
||||
@renderer
|
||||
def nodeid(self, req, tag):
|
||||
return tag(idlib.nodeid_b2a(self._storage.my_nodeid))
|
||||
|
||||
def render_storage_running(self, ctx, storage):
|
||||
if storage:
|
||||
return ctx.tag
|
||||
else:
|
||||
return T.h1["No Storage Server Running"]
|
||||
def _get_storage_stat(self, key):
|
||||
"""Get storage server statistics.
|
||||
|
||||
def render_bool(self, ctx, data):
|
||||
return {True: "Yes", False: "No"}[bool(data)]
|
||||
Storage Server keeps a dict that contains various usage and
|
||||
latency statistics. The dict looks like this:
|
||||
|
||||
def render_abbrev_space(self, ctx, size):
|
||||
{
|
||||
'storage_server.accepting_immutable_shares': 1,
|
||||
'storage_server.allocated': 0,
|
||||
'storage_server.disk_avail': 106539192320,
|
||||
'storage_server.disk_free_for_nonroot': 106539192320,
|
||||
'storage_server.disk_free_for_root': 154415284224,
|
||||
'storage_server.disk_total': 941088460800,
|
||||
'storage_server.disk_used': 786673176576,
|
||||
'storage_server.latencies.add-lease.01_0_percentile': None,
|
||||
'storage_server.latencies.add-lease.10_0_percentile': None,
|
||||
...
|
||||
}
|
||||
|
||||
``StorageServer.get_stats()`` returns the above dict. Storage
|
||||
status page uses a subset of the items in the dict, concerning
|
||||
disk usage.
|
||||
|
||||
:param str key: storage server statistic we want to know.
|
||||
"""
|
||||
return self._storage.get_stats().get(key)
|
||||
|
||||
def render_abbrev_space(self, size):
|
||||
if size is None:
|
||||
return "?"
|
||||
return u"?"
|
||||
return abbreviate_space(size)
|
||||
|
||||
def render_space(self, ctx, size):
|
||||
def render_space(self, size):
|
||||
if size is None:
|
||||
return "?"
|
||||
return "%d" % size
|
||||
return u"?"
|
||||
return u"%d" % size
|
||||
|
||||
def data_stats(self, ctx, data):
|
||||
# FYI: 'data' appears to be self, rather than the StorageServer
|
||||
# object in self.original that gets passed to render_* methods. I
|
||||
# still don't understand Nevow.
|
||||
@renderer
|
||||
def storage_stats(self, req, tag):
|
||||
# Render storage status table that appears near the top of the page.
|
||||
total = self._get_storage_stat("storage_server.disk_total")
|
||||
used = self._get_storage_stat("storage_server.disk_used")
|
||||
free_root = self._get_storage_stat("storage_server.disk_free_for_root")
|
||||
free_nonroot = self._get_storage_stat("storage_server.disk_free_for_nonroot")
|
||||
reserved = self._get_storage_stat("storage_server.reserved_space")
|
||||
available = self._get_storage_stat("storage_server.disk_avail")
|
||||
|
||||
# Nevow has nevow.accessors.DictionaryContainer: Any data= directive
|
||||
# that appears in a context in which the current data is a dictionary
|
||||
# will be looked up as keys in that dictionary. So if data_stats()
|
||||
# returns a dictionary, then we can use something like this:
|
||||
#
|
||||
# <ul n:data="stats">
|
||||
# <li>disk_total: <span n:render="abbrev" n:data="disk_total" /></li>
|
||||
# </ul>
|
||||
tag.fillSlots(
|
||||
disk_total = self.render_space(total),
|
||||
disk_total_abbrev = self.render_abbrev_space(total),
|
||||
disk_used = self.render_space(used),
|
||||
disk_used_abbrev = self.render_abbrev_space(used),
|
||||
disk_free_for_root = self.render_space(free_root),
|
||||
disk_free_for_root_abbrev = self.render_abbrev_space(free_root),
|
||||
disk_free_for_nonroot = self.render_space(free_nonroot),
|
||||
disk_free_for_nonroot_abbrev = self.render_abbrev_space(free_nonroot),
|
||||
reserved_space = self.render_space(reserved),
|
||||
reserved_space_abbrev = self.render_abbrev_space(reserved),
|
||||
disk_avail = self.render_space(available),
|
||||
disk_avail_abbrev = self.render_abbrev_space(available)
|
||||
)
|
||||
return tag
|
||||
|
||||
# to use get_stats()["storage_server.disk_total"] . However,
|
||||
# DictionaryContainer does a raw d[] instead of d.get(), so any
|
||||
# missing keys will cause an error, even if the renderer can tolerate
|
||||
# None values. To overcome this, we either need a dict-like object
|
||||
# that always returns None for unknown keys, or we must pre-populate
|
||||
# our dict with those missing keys, or we should get rid of data_
|
||||
# methods that return dicts (or find some way to override Nevow's
|
||||
# handling of dictionaries).
|
||||
@renderer
|
||||
def accepting_immutable_shares(self, req, tag):
|
||||
accepting = self._get_storage_stat("storage_server.accepting_immutable_shares")
|
||||
return tag({True: "Yes", False: "No"}[bool(accepting)])
|
||||
|
||||
d = dict([ (remove_prefix(k, "storage_server."), v)
|
||||
for k,v in self.storage.get_stats().items() ])
|
||||
d.setdefault("disk_total", None)
|
||||
d.setdefault("disk_used", None)
|
||||
d.setdefault("disk_free_for_root", None)
|
||||
d.setdefault("disk_free_for_nonroot", None)
|
||||
d.setdefault("reserved_space", None)
|
||||
d.setdefault("disk_avail", None)
|
||||
return d
|
||||
|
||||
def data_last_complete_bucket_count(self, ctx, data):
|
||||
s = self.storage.bucket_counter.get_state()
|
||||
@renderer
|
||||
def last_complete_bucket_count(self, req, tag):
|
||||
s = self._storage.bucket_counter.get_state()
|
||||
count = s.get("last-complete-bucket-count")
|
||||
if count is None:
|
||||
return "Not computed yet"
|
||||
return count
|
||||
return tag("Not computed yet")
|
||||
return tag(str(count))
|
||||
|
||||
def render_count_crawler_status(self, ctx, storage):
|
||||
p = self.storage.bucket_counter.get_progress()
|
||||
return ctx.tag[self.format_crawler_progress(p)]
|
||||
@renderer
|
||||
def count_crawler_status(self, req, tag):
|
||||
p = self._storage.bucket_counter.get_progress()
|
||||
return tag(self.format_crawler_progress(p))
|
||||
|
||||
def format_crawler_progress(self, p):
|
||||
cycletime = p["estimated-time-per-cycle"]
|
||||
@ -127,56 +150,52 @@ class StorageStatus(MultiFormatPage):
|
||||
return ["Next crawl in %s" % abbreviate_time(soon),
|
||||
cycletime_s]
|
||||
|
||||
def render_lease_expiration_enabled(self, ctx, data):
|
||||
lc = self.storage.lease_checker
|
||||
if lc.expiration_enabled:
|
||||
return ctx.tag["Enabled: expired leases will be removed"]
|
||||
else:
|
||||
return ctx.tag["Disabled: scan-only mode, no leases will be removed"]
|
||||
@renderer
|
||||
def storage_running(self, req, tag):
|
||||
if self._storage:
|
||||
return tag
|
||||
return T.h1("No Storage Server Running")
|
||||
|
||||
def render_lease_expiration_mode(self, ctx, data):
|
||||
lc = self.storage.lease_checker
|
||||
@renderer
|
||||
def lease_expiration_enabled(self, req, tag):
|
||||
lc = self._storage.lease_checker
|
||||
if lc.expiration_enabled:
|
||||
return tag("Enabled: expired leases will be removed")
|
||||
else:
|
||||
return tag("Disabled: scan-only mode, no leases will be removed")
|
||||
|
||||
@renderer
|
||||
def lease_expiration_mode(self, req, tag):
|
||||
lc = self._storage.lease_checker
|
||||
if lc.mode == "age":
|
||||
if lc.override_lease_duration is None:
|
||||
ctx.tag["Leases will expire naturally, probably 31 days after "
|
||||
"creation or renewal."]
|
||||
tag("Leases will expire naturally, probably 31 days after "
|
||||
"creation or renewal.")
|
||||
else:
|
||||
ctx.tag["Leases created or last renewed more than %s ago "
|
||||
"will be considered expired."
|
||||
% abbreviate_time(lc.override_lease_duration)]
|
||||
tag("Leases created or last renewed more than %s ago "
|
||||
"will be considered expired."
|
||||
% abbreviate_time(lc.override_lease_duration))
|
||||
else:
|
||||
assert lc.mode == "cutoff-date"
|
||||
localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date))
|
||||
isoutcdate = time_format.iso_utc_date(lc.cutoff_date)
|
||||
ctx.tag["Leases created or last renewed before %s (%s) UTC "
|
||||
"will be considered expired." % (isoutcdate, localizedutcdate, )]
|
||||
tag("Leases created or last renewed before %s (%s) UTC "
|
||||
"will be considered expired."
|
||||
% (isoutcdate, localizedutcdate, ))
|
||||
if len(lc.mode) > 2:
|
||||
ctx.tag[" The following sharetypes will be expired: ",
|
||||
" ".join(sorted(lc.sharetypes_to_expire)), "."]
|
||||
return ctx.tag
|
||||
tag(" The following sharetypes will be expired: ",
|
||||
" ".join(sorted(lc.sharetypes_to_expire)), ".")
|
||||
return tag
|
||||
|
||||
def format_recovered(self, sr, a):
|
||||
def maybe(d):
|
||||
if d is None:
|
||||
return "?"
|
||||
return "%d" % d
|
||||
return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
|
||||
(maybe(sr["%s-shares" % a]),
|
||||
maybe(sr["%s-buckets" % a]),
|
||||
maybe(sr["%s-buckets-mutable" % a]),
|
||||
maybe(sr["%s-buckets-immutable" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes-mutable" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes-immutable" % a]),
|
||||
)
|
||||
|
||||
def render_lease_current_cycle_progress(self, ctx, data):
|
||||
lc = self.storage.lease_checker
|
||||
@renderer
|
||||
def lease_current_cycle_progress(self, req, tag):
|
||||
lc = self._storage.lease_checker
|
||||
p = lc.get_progress()
|
||||
return ctx.tag[self.format_crawler_progress(p)]
|
||||
return tag(self.format_crawler_progress(p))
|
||||
|
||||
def render_lease_current_cycle_results(self, ctx, data):
|
||||
lc = self.storage.lease_checker
|
||||
@renderer
|
||||
def lease_current_cycle_results(self, req, tag):
|
||||
lc = self._storage.lease_checker
|
||||
p = lc.get_progress()
|
||||
if not p["cycle-in-progress"]:
|
||||
return ""
|
||||
@ -190,7 +209,7 @@ class StorageStatus(MultiFormatPage):
|
||||
|
||||
p = T.ul()
|
||||
def add(*pieces):
|
||||
p[T.li[pieces]]
|
||||
p(T.li(pieces))
|
||||
|
||||
def maybe(d):
|
||||
if d is None:
|
||||
@ -226,29 +245,29 @@ class StorageStatus(MultiFormatPage):
|
||||
|
||||
if so_far["corrupt-shares"]:
|
||||
add("Corrupt shares:",
|
||||
T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
|
||||
T.ul( (T.li( ["SI %s shnum %d" % corrupt_share
|
||||
for corrupt_share in so_far["corrupt-shares"] ]
|
||||
]]])
|
||||
))))
|
||||
return tag("Current cycle:", p)
|
||||
|
||||
return ctx.tag["Current cycle:", p]
|
||||
|
||||
def render_lease_last_cycle_results(self, ctx, data):
|
||||
lc = self.storage.lease_checker
|
||||
@renderer
|
||||
def lease_last_cycle_results(self, req, tag):
|
||||
lc = self._storage.lease_checker
|
||||
h = lc.get_state()["history"]
|
||||
if not h:
|
||||
return ""
|
||||
last = h[max(h.keys())]
|
||||
|
||||
start, end = last["cycle-start-finish-times"]
|
||||
ctx.tag["Last complete cycle (which took %s and finished %s ago)"
|
||||
" recovered: " % (abbreviate_time(end-start),
|
||||
abbreviate_time(time.time() - end)),
|
||||
self.format_recovered(last["space-recovered"], "actual")
|
||||
]
|
||||
tag("Last complete cycle (which took %s and finished %s ago)"
|
||||
" recovered: " % (abbreviate_time(end-start),
|
||||
abbreviate_time(time.time() - end)),
|
||||
self.format_recovered(last["space-recovered"], "actual"))
|
||||
|
||||
p = T.ul()
|
||||
|
||||
def add(*pieces):
|
||||
p[T.li[pieces]]
|
||||
p(T.li(pieces))
|
||||
|
||||
saw = self.format_recovered(last["space-recovered"], "examined")
|
||||
add("and saw a total of ", saw)
|
||||
@ -260,8 +279,42 @@ class StorageStatus(MultiFormatPage):
|
||||
|
||||
if last["corrupt-shares"]:
|
||||
add("Corrupt shares:",
|
||||
T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share
|
||||
T.ul( (T.li( ["SI %s shnum %d" % corrupt_share
|
||||
for corrupt_share in last["corrupt-shares"] ]
|
||||
]]])
|
||||
))))
|
||||
|
||||
return ctx.tag[p]
|
||||
return tag(p)
|
||||
|
||||
@staticmethod
|
||||
def format_recovered(sr, a):
|
||||
def maybe(d):
|
||||
if d is None:
|
||||
return "?"
|
||||
return "%d" % d
|
||||
return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \
|
||||
(maybe(sr["%s-shares" % a]),
|
||||
maybe(sr["%s-buckets" % a]),
|
||||
maybe(sr["%s-buckets-mutable" % a]),
|
||||
maybe(sr["%s-buckets-immutable" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes-mutable" % a]),
|
||||
abbreviate_space(sr["%s-diskbytes-immutable" % a]),
|
||||
)
|
||||
|
||||
class StorageStatus(MultiFormatResource):
|
||||
def __init__(self, storage, nickname=""):
|
||||
super(StorageStatus, self).__init__()
|
||||
self._storage = storage
|
||||
self._nickname = nickname
|
||||
|
||||
def render_HTML(self, req):
|
||||
return renderElement(req, StorageStatusElement(self._storage, self._nickname))
|
||||
|
||||
def render_JSON(self, req):
|
||||
req.setHeader("content-type", "text/plain")
|
||||
d = {"stats": self._storage.get_stats(),
|
||||
"bucket-counter": self._storage.bucket_counter.get_state(),
|
||||
"lease-checker": self._storage.lease_checker.get_state(),
|
||||
"lease-checker-progress": self._storage.lease_checker.get_progress(),
|
||||
}
|
||||
return json.dumps(d, indent=1) + "\n"
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
<html xmlns:n="http://nevow.com/ns/nevow/0.1">
|
||||
<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
|
||||
<head>
|
||||
<title>Tahoe-LAFS - Storage Server Status</title>
|
||||
<link href="/tahoe.css" rel="stylesheet" type="text/css"/>
|
||||
@ -7,19 +7,19 @@
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div n:render="storage_running">
|
||||
<div t:render="storage_running">
|
||||
|
||||
<h1>Storage Server Status</h1>
|
||||
|
||||
<table n:data="stats">
|
||||
<table class="storage_status" t:render="storage_stats">
|
||||
<tr><td>Total disk space:</td>
|
||||
<td><span n:render="abbrev_space" n:data="disk_total" /></td>
|
||||
<td>(<span n:render="space" n:data="disk_total" />)</td>
|
||||
<td><t:slot name="disk_total_abbrev" /></td>
|
||||
<td>(<t:slot name="disk_total" />)</td>
|
||||
<td />
|
||||
</tr>
|
||||
<tr><td>Disk space used:</td>
|
||||
<td>- <span n:render="abbrev_space" n:data="disk_used" /></td>
|
||||
<td>(<span n:render="space" n:data="disk_used" />)</td>
|
||||
<td>- <t:slot name="disk_used_abbrev" /></td>
|
||||
<td>(<t:slot name="disk_used" />)</td>
|
||||
<td />
|
||||
</tr>
|
||||
<tr><td />
|
||||
@ -28,18 +28,18 @@
|
||||
<td />
|
||||
</tr>
|
||||
<tr><td>Disk space free (root):</td>
|
||||
<td><span n:render="abbrev_space" n:data="disk_free_for_root"/></td>
|
||||
<td>(<span n:render="space" n:data="disk_free_for_root"/>)</td>
|
||||
<td><t:slot name="disk_free_for_root_abbrev"/></td>
|
||||
<td>(<t:slot name="disk_free_for_root"/>)</td>
|
||||
<td>[see 1]</td>
|
||||
</tr>
|
||||
<tr><td>Disk space free (non-root):</td>
|
||||
<td><span n:render="abbrev_space" n:data="disk_free_for_nonroot" /></td>
|
||||
<td>(<span n:render="space" n:data="disk_free_for_nonroot" />)</td>
|
||||
<td><t:slot name="disk_free_for_nonroot_abbrev" /></td>
|
||||
<td>(<t:slot name="disk_free_for_nonroot" />)</td>
|
||||
<td>[see 2]</td>
|
||||
</tr>
|
||||
<tr><td>Reserved space:</td>
|
||||
<td>- <span n:render="abbrev_space" n:data="reserved_space" /></td>
|
||||
<td>(<span n:render="space" n:data="reserved_space" />)</td>
|
||||
<td>- <t:slot name="reserved_space_abbrev" /></td>
|
||||
<td>(<t:slot name="reserved_space" />)</td>
|
||||
<td />
|
||||
</tr>
|
||||
<tr><td />
|
||||
@ -48,23 +48,23 @@
|
||||
<td />
|
||||
</tr>
|
||||
<tr><td>Space Available to Tahoe:</td>
|
||||
<td><span n:render="abbrev_space" n:data="disk_avail" /></td>
|
||||
<td>(<span n:render="space" n:data="disk_avail" />)</td>
|
||||
<td><t:slot name="disk_avail_abbrev" /></td>
|
||||
<td>(<t:slot name="disk_avail" />)</td>
|
||||
<td />
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<ul>
|
||||
<li>Server Nickname: <span class="nickname mine" n:render="data" n:data="nickname" /></li>
|
||||
<li>Server Nodeid: <span class="nodeid mine data-chars" n:render="string" n:data="nodeid" /></li>
|
||||
<li n:data="stats">Accepting new shares:
|
||||
<span n:render="bool" n:data="accepting_immutable_shares" /></li>
|
||||
<li>Server Nickname: <span class="nickname mine"><t:transparent t:render="nickname" /></span></li>
|
||||
<li>Server Nodeid: <span class="nodeid mine data-chars"> <t:transparent t:render="nodeid" /></span></li>
|
||||
<li>Accepting new shares:
|
||||
<span t:render="accepting_immutable_shares" /></li>
|
||||
<li>Total buckets:
|
||||
<span n:render="string" n:data="last_complete_bucket_count" />
|
||||
<span t:render="last_complete_bucket_count" />
|
||||
(the number of files and directories for which this server is holding
|
||||
a share)
|
||||
<ul>
|
||||
<li n:render="count_crawler_status" />
|
||||
<li><span t:render="count_crawler_status" /></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
@ -72,11 +72,11 @@
|
||||
<h2>Lease Expiration Crawler</h2>
|
||||
|
||||
<ul>
|
||||
<li>Expiration <span n:render="lease_expiration_enabled" /></li>
|
||||
<li n:render="lease_expiration_mode" />
|
||||
<li n:render="lease_current_cycle_progress" />
|
||||
<li n:render="lease_current_cycle_results" />
|
||||
<li n:render="lease_last_cycle_results" />
|
||||
<li>Expiration <span t:render="lease_expiration_enabled" /></li>
|
||||
<li t:render="lease_expiration_mode" />
|
||||
<li t:render="lease_current_cycle_progress" />
|
||||
<li t:render="lease_current_cycle_results" />
|
||||
<li t:render="lease_last_cycle_results" />
|
||||
</ul>
|
||||
|
||||
<hr />
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user