Merge pull request #722 from sajith/3254.status-status-nevow-to-twisted-web

Replace nevow with twisted.web.template in status.Status

Fixes: ticket:3254
This commit is contained in:
Sajith Sasidharan 2020-07-20 11:28:18 -04:00 committed by GitHub
commit e145c7b00d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 437 additions and 131 deletions

View File

@ -219,23 +219,21 @@ def test_status(alice):
found_upload = False found_upload = False
found_download = False found_download = False
for href in hrefs: for href in hrefs:
if href.startswith(u"/") or not href: if href == u"/" or not href:
continue continue
resp = requests.get( resp = requests.get(util.node_url(alice.node_dir, href))
util.node_url(alice.node_dir, u"status/{}".format(href)), if href.startswith(u"/status/up"):
)
if href.startswith(u'up'):
assert "File Upload Status" in resp.content assert "File Upload Status" in resp.content
if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
found_upload = True found_upload = True
elif href.startswith(u'down'): elif href.startswith(u"/status/down"):
assert "File Download Status" in resp.content assert "File Download Status" in resp.content
if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content: if "Total Size: {}".format(len(FILE_CONTENTS)) in resp.content:
found_download = True found_download = True
# download the specialized event information # download the specialized event information
resp = requests.get( resp = requests.get(
util.node_url(alice.node_dir, u"status/{}/event_json".format(href)), util.node_url(alice.node_dir, u"{}/event_json".format(href)),
) )
js = json.loads(resp.content) js = json.loads(resp.content)
# there's usually just one "read" operation, but this can handle many .. # there's usually just one "read" operation, but this can handle many ..

0
newsfragments/3254.minor Normal file
View File

View File

@ -4,6 +4,12 @@ Tests for ```allmydata.web.status```.
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from twisted.web.template import flattenString from twisted.web.template import flattenString
from allmydata.web.status import (
Status,
StatusElement,
)
from zope.interface import implementer from zope.interface import implementer
from allmydata.interfaces import IDownloadResults from allmydata.interfaces import IDownloadResults
@ -16,6 +22,61 @@ from .common import (
) )
from ..common import TrialTestCase from ..common import TrialTestCase
from .test_web import FakeHistory
# Test that status.StatusElement can render HTML.
class StatusTests(TrialTestCase):
def _render_status_page(self, active, recent):
elem = StatusElement(active, recent)
d = flattenString(None, elem)
return self.successResultOf(d)
def test_status_page(self):
status = Status(FakeHistory())
doc = self._render_status_page(
status._get_active_operations(),
status._get_recent_operations()
)
soup = BeautifulSoup(doc, 'html5lib')
assert_soup_has_favicon(self, soup)
assert_soup_has_tag_with_content(
self, soup, u"title",
u"Tahoe-LAFS - Recent and Active Operations"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
u"Active Operations:"
)
assert_soup_has_tag_with_content(
self, soup, u"td",
u"retrieve"
)
assert_soup_has_tag_with_content(
self, soup, u"td",
u"publish"
)
assert_soup_has_tag_with_content(
self, soup, u"td",
u"download"
)
assert_soup_has_tag_with_content(
self, soup, u"td",
u"upload"
)
assert_soup_has_tag_with_content(
self, soup, u"h2",
"Recent Operations:"
)
@implementer(IDownloadResults) @implementer(IDownloadResults)
class FakeDownloadResults(object): class FakeDownloadResults(object):

View File

@ -971,11 +971,11 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
d = self.GET("/status", followRedirect=True) d = self.GET("/status", followRedirect=True)
def _check(res): def _check(res):
self.failUnlessIn('Recent and Active Operations', res) self.failUnlessIn('Recent and Active Operations', res)
self.failUnlessIn('"down-%d"' % dl_num, res) self.failUnlessIn('"/status/down-%d"' % dl_num, res)
self.failUnlessIn('"up-%d"' % ul_num, res) self.failUnlessIn('"/status/up-%d"' % ul_num, res)
self.failUnlessIn('"mapupdate-%d"' % mu_num, res) self.failUnlessIn('"/status/mapupdate-%d"' % mu_num, res)
self.failUnlessIn('"publish-%d"' % pub_num, res) self.failUnlessIn('"/status/publish-%d"' % pub_num, res)
self.failUnlessIn('"retrieve-%d"' % ret_num, res) self.failUnlessIn('"/status/retrieve-%d"' % ret_num, res)
d.addCallback(_check) d.addCallback(_check)
d.addCallback(lambda res: self.GET("/status/?t=json")) d.addCallback(lambda res: self.GET("/status/?t=json"))
def _check_json(res): def _check_json(res):
@ -1034,6 +1034,210 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
return d return d
def test_status_path_nodash_error(self):
"""
Expect an error, because path is expected to be of the form
"/status/{up,down,..}-%number", with a hyphen.
"""
return self.shouldFail2(error.Error,
"test_status_path_nodash",
"400 Bad Request",
"no '-' in 'nodash'",
self.GET,
"/status/nodash")
def test_status_page_contains_links(self):
"""
Check that the rendered `/status` page contains all the
expected links.
"""
def _check_status_page_links(response):
(body, status, _) = response
self.failUnlessReallyEqual(int(status), 200)
soup = BeautifulSoup(body, 'html5lib')
h = self.s.get_history()
# Check for `<a href="/status/retrieve-0">Not started</a>`
ret_num = h.list_all_retrieve_statuses()[0].get_counter()
assert_soup_has_tag_with_attributes_and_content(
self, soup, u"a",
u"Not started",
{u"href": u"/status/retrieve-{}".format(ret_num)}
)
# Check for `<a href="/status/publish-0">Not started</a></td>`
pub_num = h.list_all_publish_statuses()[0].get_counter()
assert_soup_has_tag_with_attributes_and_content(
self, soup, u"a",
u"Not started",
{u"href": u"/status/publish-{}".format(pub_num)}
)
# Check for `<a href="/status/mapupdate-0">Not started</a>`
mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
assert_soup_has_tag_with_attributes_and_content(
self, soup, u"a",
u"Not started",
{u"href": u"/status/mapupdate-{}".format(mu_num)}
)
# Check for `<a href="/status/down-0">fetching segments
# 2,3; errors on segment 1</a>`: see build_one_ds() above.
dl_num = h.list_all_download_statuses()[0].get_counter()
assert_soup_has_tag_with_attributes_and_content(
self, soup, u"a",
u"fetching segments 2,3; errors on segment 1",
{u"href": u"/status/down-{}".format(dl_num)}
)
# Check for `<a href="/status/up-0">Not started</a>`
ul_num = h.list_all_upload_statuses()[0].get_counter()
assert_soup_has_tag_with_attributes_and_content(
self, soup, u"a",
u"Not started",
{u"href": u"/status/up-{}".format(ul_num)}
)
d = self.GET("/status", return_response=True)
d.addCallback(_check_status_page_links)
return d
def test_status_path_trailing_slashes(self):
"""
Test that both `GET /status` and `GET /status/` are treated
alike, but reject any additional trailing slashes and other
non-existent child nodes.
"""
def _check_status(response):
(body, status, _) = response
self.failUnlessReallyEqual(int(status), 200)
soup = BeautifulSoup(body, 'html5lib')
assert_soup_has_favicon(self, soup)
assert_soup_has_tag_with_content(
self, soup, u"title",
u"Tahoe-LAFS - Recent and Active Operations"
)
d = self.GET("/status", return_response=True)
d.addCallback(_check_status)
d = self.GET("/status/", return_response=True)
d.addCallback(_check_status)
d = self.shouldFail2(error.Error,
"test_status_path_trailing_slashes",
"400 Bad Request",
"no '-' in ''",
self.GET,
"/status//")
d = self.shouldFail2(error.Error,
"test_status_path_trailing_slashes",
"400 Bad Request",
"no '-' in ''",
self.GET,
"/status////////")
return d
def test_status_path_404_error(self):
"""
Looking for non-existent statuses under child paths should
exercises all the iterators in web.status.Status.getChild().
The test suite (hopefully!) would not have done any setup for
a very large number of statuses at this point, now or in the
future, so these all should always return 404.
"""
d = self.GET("/status/up-9999999")
d.addBoth(self.should404, "test_status_path_404_error (up)")
d = self.GET("/status/down-9999999")
d.addBoth(self.should404, "test_status_path_404_error (down)")
d = self.GET("/status/mapupdate-9999999")
d.addBoth(self.should404, "test_status_path_404_error (mapupdate)")
d = self.GET("/status/publish-9999999")
d.addBoth(self.should404, "test_status_path_404_error (publish)")
d = self.GET("/status/retrieve-9999999")
d.addBoth(self.should404, "test_status_path_404_error (retrieve)")
return d
def _check_status_subpath_result(self, result, expected_title):
"""
Helper to verify that results of "GET /status/up-0" and
similar are as expected.
"""
body, status, _ = result
self.failUnlessReallyEqual(int(status), 200)
soup = BeautifulSoup(body, 'html5lib')
assert_soup_has_favicon(self, soup)
assert_soup_has_tag_with_content(
self, soup, u"title", expected_title
)
def test_status_up_subpath(self):
"""
See that "GET /status/up-0" works.
"""
h = self.s.get_history()
ul_num = h.list_all_upload_statuses()[0].get_counter()
d = self.GET("/status/up-{}".format(ul_num), return_response=True)
d.addCallback(self._check_status_subpath_result,
u"Tahoe-LAFS - File Upload Status")
return d
def test_status_down_subpath(self):
"""
See that "GET /status/down-0" works.
"""
h = self.s.get_history()
dl_num = h.list_all_download_statuses()[0].get_counter()
d = self.GET("/status/down-{}".format(dl_num), return_response=True)
d.addCallback(self._check_status_subpath_result,
u"Tahoe-LAFS - File Download Status")
return d
def test_status_mapupdate_subpath(self):
"""
See that "GET /status/mapupdate-0" works.
"""
h = self.s.get_history()
mu_num = h.list_all_mapupdate_statuses()[0].get_counter()
d = self.GET("/status/mapupdate-{}".format(mu_num), return_response=True)
d.addCallback(self._check_status_subpath_result,
u"Tahoe-LAFS - Mutable File Servermap Update Status")
return d
def test_status_publish_subpath(self):
"""
See that "GET /status/publish-0" works.
"""
h = self.s.get_history()
pub_num = h.list_all_publish_statuses()[0].get_counter()
d = self.GET("/status/publish-{}".format(pub_num), return_response=True)
d.addCallback(self._check_status_subpath_result,
u"Tahoe-LAFS - Mutable File Publish Status")
return d
def test_status_retrieve_subpath(self):
"""
See that "GET /status/retrieve-0" works.
"""
h = self.s.get_history()
ret_num = h.list_all_retrieve_statuses()[0].get_counter()
d = self.GET("/status/retrieve-{}".format(ret_num), return_response=True)
d.addCallback(self._check_status_subpath_result,
u"Tahoe-LAFS - Mutable File Retrieve Status")
return d
def test_GET_FILEURL(self): def test_GET_FILEURL(self):
d = self.GET(self.public_url + "/foo/bar.txt") d = self.GET(self.public_url + "/foo/bar.txt")
d.addCallback(self.failUnlessIsBarDotTxt) d.addCallback(self.failUnlessIsBarDotTxt)

View File

@ -1,5 +1,7 @@
import pprint, itertools, hashlib import pprint
import itertools
import hashlib
import json import json
from twisted.internet import defer from twisted.internet import defer
from twisted.python.filepath import FilePath from twisted.python.filepath import FilePath
@ -11,21 +13,26 @@ from twisted.web.template import (
renderElement, renderElement,
tags, tags,
) )
from nevow import rend, tags as T
from allmydata.util import base32, idlib from allmydata.util import base32, idlib
from allmydata.web.common import ( from allmydata.web.common import (
getxmlfile,
abbreviate_time, abbreviate_time,
abbreviate_rate, abbreviate_rate,
abbreviate_size, abbreviate_size,
plural, plural,
compute_rate, compute_rate,
render_time, render_time,
MultiFormatPage,
MultiFormatResource, MultiFormatResource,
SlotsSequenceElement,
WebError,
)
from allmydata.interfaces import (
IUploadStatus,
IDownloadStatus,
IPublishStatus,
IRetrieveStatus,
IServermapUpdaterStatus,
) )
from allmydata.interfaces import IUploadStatus, IDownloadStatus, \
IPublishStatus, IRetrieveStatus, IServermapUpdaterStatus
class UploadResultsRendererMixin(Element): class UploadResultsRendererMixin(Element):
@ -1231,14 +1238,21 @@ def marshal_json(s):
return item return item
class Status(MultiFormatPage): class Status(MultiFormatResource):
docFactory = getxmlfile("status.xhtml") """Renders /status page."""
addSlash = True
def __init__(self, history): def __init__(self, history):
rend.Page.__init__(self, history) """
:param allmydata.history.History history: provides operation statuses.
"""
super(Status, self).__init__()
self.history = history self.history = history
def render_HTML(self, req):
elem = StatusElement(self._get_active_operations(),
self._get_recent_operations())
return renderElement(req, elem)
def render_JSON(self, req): def render_JSON(self, req):
# modern browsers now render this instead of forcing downloads # modern browsers now render this instead of forcing downloads
req.setHeader("content-type", "application/json") req.setHeader("content-type", "application/json")
@ -1254,97 +1268,23 @@ class Status(MultiFormatPage):
return json.dumps(data, indent=1) + "\n" return json.dumps(data, indent=1) + "\n"
def _get_all_statuses(self): def getChild(self, path, request):
h = self.history # The "if (path is empty) return self" line should handle
return itertools.chain(h.list_all_upload_statuses(), # trailing slash in request path.
h.list_all_download_statuses(), #
h.list_all_mapupdate_statuses(), # Twisted Web's documentation says this: "If the URL ends in a
h.list_all_publish_statuses(), # slash, for example ``http://example.com/foo/bar/`` , the
h.list_all_retrieve_statuses(), # final URL segment will be an empty string. Resources can
h.list_all_helper_statuses(), # thus know if they were requested with or without a final
) # slash."
if not path and request.postpath != ['']:
return self
def data_active_operations(self, ctx, data):
return self._get_active_operations()
def _get_active_operations(self):
active = [s
for s in self._get_all_statuses()
if s.get_active()]
active.sort(lambda a, b: cmp(a.get_started(), b.get_started()))
active.reverse()
return active
def data_recent_operations(self, ctx, data):
return self._get_recent_operations()
def _get_recent_operations(self):
recent = [s
for s in self._get_all_statuses()
if not s.get_active()]
recent.sort(lambda a, b: cmp(a.get_started(), b.get_started()))
recent.reverse()
return recent
def render_row(self, ctx, data):
s = data
started_s = render_time(s.get_started())
ctx.fillSlots("started", started_s)
si_s = base32.b2a_or_none(s.get_storage_index())
if si_s is None:
si_s = "(None)"
ctx.fillSlots("si", si_s)
ctx.fillSlots("helper", {True: "Yes",
False: "No"}[s.using_helper()])
size = s.get_size()
if size is None:
size = "(unknown)"
elif isinstance(size, (int, long, float)):
size = abbreviate_size(size)
ctx.fillSlots("total_size", size)
progress = data.get_progress()
if IUploadStatus.providedBy(data):
link = "up-%d" % data.get_counter()
ctx.fillSlots("type", "upload")
# TODO: make an ascii-art bar
(chk, ciphertext, encandpush) = progress
progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" %
( (100.0 * chk),
(100.0 * ciphertext),
(100.0 * encandpush) ))
ctx.fillSlots("progress", progress_s)
elif IDownloadStatus.providedBy(data):
link = "down-%d" % data.get_counter()
ctx.fillSlots("type", "download")
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
elif IPublishStatus.providedBy(data):
link = "publish-%d" % data.get_counter()
ctx.fillSlots("type", "publish")
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
elif IRetrieveStatus.providedBy(data):
ctx.fillSlots("type", "retrieve")
link = "retrieve-%d" % data.get_counter()
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
else:
assert IServermapUpdaterStatus.providedBy(data)
ctx.fillSlots("type", "mapupdate %s" % data.get_mode())
link = "mapupdate-%d" % data.get_counter()
ctx.fillSlots("progress", "%.1f%%" % (100.0 * progress))
ctx.fillSlots("status", T.a(href=link)[s.get_status()])
return ctx.tag
def childFactory(self, ctx, name):
h = self.history h = self.history
try: try:
stype, count_s = name.split("-") stype, count_s = path.split("-")
except ValueError: except ValueError:
raise RuntimeError( raise WebError("no '-' in '{}'".format(path))
"no - in '{}'".format(name)
)
count = int(count_s) count = int(count_s)
if stype == "up": if stype == "up":
for s in itertools.chain(h.list_all_upload_statuses(), for s in itertools.chain(h.list_all_upload_statuses(),
@ -1370,6 +1310,109 @@ class Status(MultiFormatPage):
if s.get_counter() == count: if s.get_counter() == count:
return RetrieveStatusPage(s) return RetrieveStatusPage(s)
def _get_all_statuses(self):
h = self.history
return itertools.chain(h.list_all_upload_statuses(),
h.list_all_download_statuses(),
h.list_all_mapupdate_statuses(),
h.list_all_publish_statuses(),
h.list_all_retrieve_statuses(),
h.list_all_helper_statuses(),
)
def _get_active_operations(self):
active = [s
for s in self._get_all_statuses()
if s.get_active()]
active.sort(lambda a, b: cmp(a.get_started(), b.get_started()))
active.reverse()
return active
def _get_recent_operations(self):
recent = [s
for s in self._get_all_statuses()
if not s.get_active()]
recent.sort(lambda a, b: cmp(a.get_started(), b.get_started()))
recent.reverse()
return recent
class StatusElement(Element):
loader = XMLFile(FilePath(__file__).sibling("status.xhtml"))
def __init__(self, active, recent):
super(StatusElement, self).__init__()
self._active = active
self._recent = recent
@renderer
def active_operations(self, req, tag):
active = [self.get_op_state(op) for op in self._active]
return SlotsSequenceElement(tag, active)
@renderer
def recent_operations(self, req, tag):
recent = [self.get_op_state(op) for op in self._recent]
return SlotsSequenceElement(tag, recent)
@staticmethod
def get_op_state(op):
result = dict()
started_s = render_time(op.get_started())
result["started"] = started_s
si_s = base32.b2a_or_none(op.get_storage_index())
if si_s is None:
si_s = "(None)"
result["si"] = si_s
result["helper"] = {True: "Yes", False: "No"}[op.using_helper()]
size = op.get_size()
if size is None:
size = "(unknown)"
elif isinstance(size, (int, long, float)):
size = abbreviate_size(size)
result["total_size"] = size
progress = op.get_progress()
if IUploadStatus.providedBy(op):
link = "up-%d" % op.get_counter()
result["type"] = "upload"
# TODO: make an ascii-art bar
(chk, ciphertext, encandpush) = progress
progress_s = ("hash: %.1f%%, ciphertext: %.1f%%, encode: %.1f%%" %
((100.0 * chk),
(100.0 * ciphertext),
(100.0 * encandpush)))
result["progress"] = progress_s
elif IDownloadStatus.providedBy(op):
link = "down-%d" % op.get_counter()
result["type"] = "download"
result["progress"] = "%.1f%%" % (100.0 * progress)
elif IPublishStatus.providedBy(op):
link = "publish-%d" % op.get_counter()
result["type"] = "publish"
result["progress"] = "%.1f%%" % (100.0 * progress)
elif IRetrieveStatus.providedBy(op):
result["type"] = "retrieve"
link = "retrieve-%d" % op.get_counter()
result["progress"] = "%.1f%%" % (100.0 * progress)
else:
assert IServermapUpdaterStatus.providedBy(op)
result["type"] = "mapupdate %s" % op.get_mode()
link = "mapupdate-%d" % op.get_counter()
result["progress"] = "%.1f%%" % (100.0 * progress)
result["status"] = tags.a(op.get_status(),
href="/status/{}".format(link))
return result
# Render "/helper_status" page. # Render "/helper_status" page.
class HelperStatus(MultiFormatResource): class HelperStatus(MultiFormatResource):

View File

@ -1,4 +1,4 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1"> <html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head> <head>
<title>Tahoe-LAFS - Recent and Active Operations</title> <title>Tahoe-LAFS - Recent and Active Operations</title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/> <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
@ -11,8 +11,8 @@
<h2>Active Operations:</h2> <h2>Active Operations:</h2>
<table align="left" class="table-headings-top" n:render="sequence" n:data="active_operations"> <table align="left" class="table-headings-top" t:render="active_operations">
<tr n:pattern="header"> <tr t:render="header">
<th>Type</th> <th>Type</th>
<th>Storage Index</th> <th>Storage Index</th>
<th>Helper?</th> <th>Helper?</th>
@ -20,21 +20,21 @@
<th>Progress</th> <th>Progress</th>
<th>Status</th> <th>Status</th>
</tr> </tr>
<tr n:pattern="item" n:render="row"> <tr t:render="item">
<td><n:slot name="type"/></td> <td><t:slot name="type"/></td>
<td><n:slot name="si"/></td> <td><t:slot name="si"/></td>
<td><n:slot name="helper"/></td> <td><t:slot name="helper"/></td>
<td><n:slot name="total_size"/></td> <td><t:slot name="total_size"/></td>
<td><n:slot name="progress"/></td> <td><t:slot name="progress"/></td>
<td><n:slot name="status"/></td> <td><t:slot name="status"/></td>
</tr> </tr>
<tr n:pattern="empty"><td>No active operations!</td></tr> <tr t:render="empty"><td>No active operations!</td></tr>
</table> </table>
<br clear="all" /> <br clear="all" />
<h2>Recent Operations:</h2> <h2>Recent Operations:</h2>
<table align="left" class="table-headings-top" n:render="sequence" n:data="recent_operations"> <table align="left" class="table-headings-top" t:render="recent_operations">
<tr n:pattern="header"> <tr t:render="header">
<th>Started</th> <th>Started</th>
<th>Type</th> <th>Type</th>
<th>Storage Index</th> <th>Storage Index</th>
@ -43,16 +43,16 @@
<th>Progress</th> <th>Progress</th>
<th>Status</th> <th>Status</th>
</tr> </tr>
<tr n:pattern="item" n:render="row"> <tr t:render="item">
<td><n:slot name="started"/></td> <td><t:slot name="started"/></td>
<td><n:slot name="type"/></td> <td><t:slot name="type"/></td>
<td><n:slot name="si"/></td> <td><t:slot name="si"/></td>
<td><n:slot name="helper"/></td> <td><t:slot name="helper"/></td>
<td><n:slot name="total_size"/></td> <td><t:slot name="total_size"/></td>
<td><n:slot name="progress"/></td> <td><t:slot name="progress"/></td>
<td><n:slot name="status"/></td> <td><t:slot name="status"/></td>
</tr> </tr>
<tr n:pattern="empty"><td>No recent operations!</td></tr> <tr t:render="empty"><td>No recent operations!</td></tr>
</table> </table>
<br clear="all" /> <br clear="all" />