ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-06-11 20:11:47 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

helper: add more stats to webapi, at /helper_status

Browse Source
This commit is contained in:
Brian Warner
2008-03-27 16:46:08 -07:00
parent 36f5c025a6
commit 2c96a32633
5 changed files with 117 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
misc/munin/tahoe-helperstats.py Normal file
View File

@ -0,0 +1,25 @@
#! /usr/bin/python
import os, sys
import urllib
import simplejson
configinfo = """\
graph_title Tahoe Helper Stats - Bytes Fetched
graph_vlabel bytes
graph_info This graph shows the amount of data being fetched by the helper
fetched.label Bytes Fetched
fetched.type GAUGE
fetched.draw LINE1
fetched.min 0
"""
if len(sys.argv) > 1:
if sys.argv[1] == "config":
print configinfo.rstrip()
sys.exit(0)
url = os.environ["url"]
data = simplejson.loads(urllib.urlopen(url).read())
print "fetched.value %d" % data["CHK_fetched_bytes"]

14
src/allmydata/offloaded.py
View File

@ -205,12 +205,12 @@ class CHKUploadHelper(Referenceable, upload.CHKUploader):
self._reader.close() self._reader.close()
os.unlink(self._encoding_file) os.unlink(self._encoding_file)
self._finished_observers.fire(r) self._finished_observers.fire(r)
self._helper.upload_finished(self._storage_index) self._helper.upload_finished(self._storage_index, size)
del self._reader del self._reader
def _failed(self, f): def _failed(self, f):
self._finished_observers.fire(f) self._finished_observers.fire(f)
self._helper.upload_finished(self._storage_index) self._helper.upload_finished(self._storage_index, 0)
del self._reader del self._reader
class AskUntilSuccessMixin: class AskUntilSuccessMixin:
@ -377,6 +377,7 @@ class CHKCiphertextFetcher(AskUntilSuccessMixin):
self._f.write(data) self._f.write(data)
self._have += len(data) self._have += len(data)
self._ciphertext_fetched += len(data) self._ciphertext_fetched += len(data)
self._upload_helper._helper._stats["CHK_fetched_bytes"] += len(data)
return False # not done return False # not done
d.addCallback(_got_data) d.addCallback(_got_data)
return d return d
@ -476,6 +477,8 @@ class Helper(Referenceable, service.MultiService):
self._stats = {"CHK_upload_requests": 0, self._stats = {"CHK_upload_requests": 0,
"CHK_upload_already_present": 0, "CHK_upload_already_present": 0,
"CHK_upload_need_upload": 0, "CHK_upload_need_upload": 0,
"CHK_fetched_bytes": 0,
"CHK_encoded_bytes": 0,
} }
service.MultiService.__init__(self) service.MultiService.__init__(self)
@ -491,11 +494,11 @@ class Helper(Referenceable, service.MultiService):
for fn in os.listdir(self._chk_incoming): for fn in os.listdir(self._chk_incoming):
size = os.stat(os.path.join(self._chk_incoming, fn))[stat.ST_SIZE] size = os.stat(os.path.join(self._chk_incoming, fn))[stat.ST_SIZE]
chk_incoming_files += 1 chk_incoming_files += 1
chk_incoming_size += 1 chk_incoming_size += size
for fn in os.listdir(self._chk_encoding): for fn in os.listdir(self._chk_encoding):
size = os.stat(os.path.join(self._chk_encoding, fn))[stat.ST_SIZE] size = os.stat(os.path.join(self._chk_encoding, fn))[stat.ST_SIZE]
chk_encoding_files += 1 chk_encoding_files += 1
chk_encoding_size += 1 chk_encoding_size += size
stats = {"CHK_active_uploads": len(self._active_uploads), stats = {"CHK_active_uploads": len(self._active_uploads),
"CHK_incoming_files": chk_incoming_files, "CHK_incoming_files": chk_incoming_files,
"CHK_incoming_size": chk_incoming_size, "CHK_incoming_size": chk_incoming_size,
@ -583,5 +586,6 @@ class Helper(Referenceable, service.MultiService):
d.addCallback(_checked) d.addCallback(_checked)
return d return d
def upload_finished(self, storage_index): def upload_finished(self, storage_index, size):
self._stats["CHK_encoded_bytes"] += size
del self._active_uploads[storage_index] del self._active_uploads[storage_index]

32
src/allmydata/web/helper.xhtml Normal file
View File

@ -0,0 +1,32 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1">
<head>
<title>Helper Status - AllMyData Tahoe</title>
<!-- <link href="http://www.allmydata.com/common/css/styles.css"
rel="stylesheet" type="text/css"/> -->
<link href="/webform_css" rel="stylesheet" type="text/css"/>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<h1>Helper Status</h1>
<h2>Immutable Uploads</h2>
<ul>
<li>Active: <span n:render="active_uploads" /></li>
<li>--</li>
<li>Bytes Fetched: <span n:render="upload_bytes_fetched" /></li>
<li>Incoming: <span n:render="incoming" /></li>
<li>Encoding: <span n:render="encoding" /></li>
<li>Bytes Encoded: <span n:render="upload_bytes_encoded" /></li>
<li>--</li>
<li>Total Requests: <span n:render="upload_requests" /></li>
<ul>
<li>Already Present: <span n:render="upload_already_present" /></li>
<li>Need Upload: <span n:render="upload_need_upload" /></li>
</ul>
</ul>
<div>Return to the <a href="/">Welcome Page</a></div>
</body>
</html>

52
src/allmydata/web/status.py
View File

@ -1,10 +1,11 @@
import time import time
import simplejson
from twisted.internet import defer from twisted.internet import defer
from nevow import rend, tags as T from nevow import rend, inevow, tags as T
from allmydata.util import base32, idlib from allmydata.util import base32, idlib
from allmydata.web.common import IClient, getxmlfile, abbreviate_time, \ from allmydata.web.common import IClient, getxmlfile, abbreviate_time, \
abbreviate_rate abbreviate_rate, get_arg
from allmydata.interfaces import IUploadStatus, IDownloadStatus, \ from allmydata.interfaces import IUploadStatus, IDownloadStatus, \
IPublishStatus, IRetrieveStatus IPublishStatus, IRetrieveStatus
@ -760,3 +761,50 @@ class Status(rend.Page):
return RetrieveStatusPage(s) return RetrieveStatusPage(s)
class HelperStatus(rend.Page):
docFactory = getxmlfile("helper.xhtml")
def renderHTTP(self, ctx):
t = get_arg(inevow.IRequest(ctx), "t")
if t == "json":
return self.render_JSON(ctx)
# is there a better way to provide 'data' to all rendering methods?
helper = IClient(ctx).getServiceNamed("helper")
self.original = helper.get_stats()["helper"]
return rend.Page.renderHTTP(self, ctx)
def render_JSON(self, ctx):
try:
h = IClient(ctx).getServiceNamed("helper")
except KeyError:
return simplejson.dumps({})
stats = h.get_stats()["helper"]
return simplejson.dumps(stats, indent=1)
def render_active_uploads(self, ctx, data):
return data["CHK_active_uploads"]
def render_incoming(self, ctx, data):
return "%d bytes in %d files" % (data["CHK_incoming_size"],
data["CHK_incoming_files"])
def render_encoding(self, ctx, data):
return "%d bytes in %d files" % (data["CHK_encoding_size"],
data["CHK_encoding_files"])
def render_upload_requests(self, ctx, data):
return str(data["CHK_upload_requests"])
def render_upload_already_present(self, ctx, data):
return str(data["CHK_upload_already_present"])
def render_upload_need_upload(self, ctx, data):
return str(data["CHK_upload_need_upload"])
def render_upload_bytes_fetched(self, ctx, data):
return str(data["CHK_fetched_bytes"])
def render_upload_bytes_encoded(self, ctx, data):
return str(data["CHK_encoded_bytes"])

1
src/allmydata/webish.py
View File

@ -1445,6 +1445,7 @@ class Root(rend.Page):
child_provisioning = provisioning.ProvisioningTool() child_provisioning = provisioning.ProvisioningTool()
child_status = status.Status() child_status = status.Status()
child_helper_status = status.HelperStatus()
def data_version(self, ctx, data): def data_version(self, ctx, data):
return get_package_versions_string() return get_package_versions_string()