Merge pull request from meejah/ticket3252-port-web-directory.manifest.2

3263 port web directory.manifest.2
This commit is contained in:
meejah 2019-12-21 23:50:46 +00:00 committed by GitHub
commit 5f68190363
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 216 additions and 66 deletions

0
newsfragments/3263.minor Normal file
View File

View File

@ -280,6 +280,13 @@ class NoNetworkGrid(service.MultiService):
def __init__(self, basedir, num_clients, num_servers,
client_config_hooks, port_assigner):
service.MultiService.__init__(self)
# We really need to get rid of this pattern here (and
# everywhere) in Tahoe where "async work" is started in
# __init__ For now, we at least keep the errors so they can
# cause tests to fail less-improperly (see _check_clients)
self._setup_errors = []
self.port_assigner = port_assigner
self.basedir = basedir
fileutil.make_dirs(basedir)
@ -300,6 +307,20 @@ class NoNetworkGrid(service.MultiService):
d = self.make_client(i)
d.addCallback(lambda c: self.clients.append(c))
def _bad(f):
self._setup_errors.append(f)
d.addErrback(_bad)
def _check_clients(self):
"""
The anti-pattern of doing async work in __init__ means we need to
check if that work completed successfully. This method either
returns nothing or raises an exception in case __init__ failed
to complete properly
"""
if self._setup_errors:
raise self._setup_errors[0].value
@defer.inlineCallbacks
def make_client(self, i, write_config=True):
clientid = hashutil.tagged_hash("clientid", str(i))[:20]
@ -364,6 +385,7 @@ class NoNetworkGrid(service.MultiService):
return self.proxies_by_id.keys()
def rebuild_serverlist(self):
self._check_clients()
self.all_servers = frozenset(self.proxies_by_id.values())
for c in self.clients:
c._servers = self.all_servers
@ -440,12 +462,14 @@ class GridTestMixin(object):
self._record_webports_and_baseurls()
def _record_webports_and_baseurls(self):
self.g._check_clients()
self.client_webports = [c.getServiceNamed("webish").getPortnum()
for c in self.g.clients]
self.client_baseurls = [c.getServiceNamed("webish").getURL()
for c in self.g.clients]
def get_client_config(self, i=0):
self.g._check_clients()
return self.g.clients[i].config
def get_clientdir(self, i=0):
@ -454,9 +478,11 @@ class GridTestMixin(object):
return self.get_client_config(i).get_config_path()
def get_client(self, i=0):
self.g._check_clients()
return self.g.clients[i]
def restart_client(self, i=0):
self.g._check_clients()
client = self.g.clients[i]
d = defer.succeed(None)
d.addCallback(lambda ign: self.g.removeService(client))

View File

@ -5,6 +5,8 @@ import json
import treq
import mock
from bs4 import BeautifulSoup
from twisted.application import service
from twisted.trial import unittest
from twisted.internet import defer
@ -49,6 +51,10 @@ from ..common import (
make_mutable_file_uri,
create_mutable_filenode,
)
from .common import (
assert_soup_has_favicon,
assert_soup_has_text,
)
from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
from allmydata.mutable import servermap, publish, retrieve
from .. import common_util as testutil
@ -2038,11 +2044,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
return d
d.addCallback(getman, None)
def _got_html(manifest):
self.failUnlessIn("Manifest of SI=", manifest)
self.failUnlessIn("<td>sub</td>", manifest)
self.failUnlessIn(self._sub_uri, manifest)
self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
self.failUnlessIn(FAVICON_MARKUP, manifest)
soup = BeautifulSoup(manifest, 'html5lib')
assert_soup_has_text(self, soup, "Manifest of SI=")
assert_soup_has_text(self, soup, "sub")
assert_soup_has_text(self, soup, self._sub_uri)
assert_soup_has_text(self, soup, "sub/baz.txt")
assert_soup_has_favicon(self, soup)
d.addCallback(_got_html)
# both t=status and unadorned GET should be identical

View File

@ -1,12 +1,22 @@
import json
import urllib
from datetime import timedelta
from zope.interface import implementer
from twisted.internet import defer
from twisted.internet.interfaces import IPushProducer
from twisted.python.failure import Failure
from twisted.web import http
from twisted.web.template import (
Element,
XMLFile,
renderElement,
renderer,
tags,
)
from hyperlink import URL
from twisted.python.filepath import FilePath
from nevow import url, rend, inevow, tags as T
from nevow.inevow import IRequest
@ -14,20 +24,40 @@ from foolscap.api import fireEventually
from allmydata.util import base32
from allmydata.util.encodingutil import to_str
from allmydata.uri import from_string_dirnode
from allmydata.uri import (
from_string_dirnode,
from_string,
CHKFileURI,
WriteableSSKFileURI,
ReadonlySSKFileURI,
)
from allmydata.interfaces import IDirectoryNode, IFileNode, IFilesystemNode, \
IImmutableFileNode, IMutableFileNode, ExistingChildError, \
NoSuchChildError, EmptyPathnameComponentError, SDMF_VERSION, MDMF_VERSION
from allmydata.blacklist import ProhibitedNode
from allmydata.monitor import Monitor, OperationCancelledError
from allmydata import dirnode
from allmydata.web.common import text_plain, WebError, \
NeedOperationHandleError, \
boolean_of_arg, get_arg, get_root, parse_replace_arg, \
should_create_intermediate_directories, \
getxmlfile, RenderMixin, humanize_failure, convert_children_json, \
get_format, get_mutable_type, get_filenode_metadata, render_time, \
MultiFormatPage
from allmydata.web.common import (
text_plain,
WebError,
NeedOperationHandleError,
boolean_of_arg,
get_arg,
get_root,
parse_replace_arg,
should_create_intermediate_directories,
getxmlfile,
RenderMixin,
humanize_failure,
convert_children_json,
get_format,
get_mutable_type,
get_filenode_metadata,
render_time,
MultiFormatPage,
MultiFormatResource,
SlotsSequenceElement,
)
from allmydata.web.filenode import ReplaceMeMixin, \
FileNodeHandler, PlaceHolderNodeHandler
from allmydata.web.check_results import CheckResultsRenderer, \
@ -943,8 +973,128 @@ class RenameForm(rend.Page):
ctx.tag.attributes['value'] = name
return ctx.tag
class ManifestResults(MultiFormatPage, ReloadMixin):
docFactory = getxmlfile("manifest.xhtml")
class ReloadableMonitorElement(Element):
"""
Like 'ReloadMixin', but for twisted.web.template style. This
provides renderers for "reload" and "refesh" and a self.monitor
attribute (which is an instance of IMonitor)
"""
refresh_time = timedelta(seconds=60)
def __init__(self, monitor):
self.monitor = monitor
super(ReloadableMonitorElement, self).__init__()
@renderer
def refresh(self, req, tag):
if self.monitor.is_finished():
return u""
tag.attributes[u"http-equiv"] = u"refresh"
tag.attributes[u"content"] = u"{}".format(self.refresh_time.seconds)
return tag
@renderer
def reload(self, req, tag):
if self.monitor.is_finished():
return u""
reload_url = URL.from_text(u"{}".format(req.path))
cancel_button = tags.form(
[
tags.input(type=u"submit", value=u"Cancel"),
],
action=reload_url.replace(query={u"t": u"cancel"}).to_uri().to_text(),
method=u"POST",
enctype=u"multipart/form-data",
)
return tag([
u"Operation still running: ",
tags.a(
u"Reload",
href=reload_url.replace(query={u"output": u"html"}).to_uri().to_text(),
),
cancel_button,
])
def _slashify_path(path):
"""
Converts a tuple from a 'manifest' path into a string with slashes
in it
"""
if not path:
return ""
return "/".join([p.encode("utf-8") for p in path])
def _cap_to_link(root, path, cap):
"""
Turns a capability-string into a WebAPI link tag
:param text root: the root piece of the URI
:param text cap: the capability-string
:returns: something suitable for `IRenderable`, specifically
either a valid local link (tags.a instance) to the capability
or an empty string.
"""
if cap:
root_url = URL.from_text(u"{}".format(root))
cap_obj = from_string(cap)
if isinstance(cap_obj, (CHKFileURI, WriteableSSKFileURI, ReadonlySSKFileURI)):
uri_link = root_url.child(
u"file",
u"{}".format(urllib.quote(cap)),
u"{}".format(urllib.quote(path[-1])),
)
else:
uri_link = root_url.child(
u"uri",
u"{}".format(urllib.quote(cap, safe="")),
)
return tags.a(cap, href=uri_link.to_text())
else:
return u""
class ManifestElement(ReloadableMonitorElement):
loader = XMLFile(FilePath(__file__).sibling("manifest.xhtml"))
def _si_abbrev(self):
si = self.monitor.origin_si
if not si:
return "<LIT>"
return base32.b2a(si)[:6]
@renderer
def title(self, req, tag):
return tag(
"Manifest of SI={}".format(self._si_abbrev())
)
@renderer
def header(self, req, tag):
return tag(
"Manifest of SI={}".format(self._si_abbrev())
)
@renderer
def items(self, req, tag):
manifest = self.monitor.get_status()["manifest"]
root = get_root(req)
rows = [
{
"path": _slashify_path(path),
"cap": _cap_to_link(root, path, cap),
}
for path, cap in manifest
]
return SlotsSequenceElement(tag, rows)
class ManifestResults(MultiFormatResource, ReloadMixin):
# Control MultiFormatPage
formatArgument = "output"
@ -954,21 +1104,19 @@ class ManifestResults(MultiFormatPage, ReloadMixin):
self.client = client
self.monitor = monitor
# The default format is HTML but the HTML renderer is just renderHTTP.
render_HTML = None
def slashify_path(self, path):
if not path:
return ""
return "/".join([p.encode("utf-8") for p in path])
def render_HTML(self, req):
return renderElement(
req,
ManifestElement(self.monitor)
)
def render_TEXT(self, req):
req.setHeader("content-type", "text/plain")
lines = []
is_finished = self.monitor.is_finished()
lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
for (path, cap) in self.monitor.get_status()["manifest"]:
lines.append(self.slashify_path(path) + " " + cap)
for path, cap in self.monitor.get_status()["manifest"]:
lines.append(_slashify_path(path) + " " + cap)
return "\n".join(lines) + "\n"
def render_JSON(self, req):
@ -1002,37 +1150,6 @@ class ManifestResults(MultiFormatPage, ReloadMixin):
# CPU.
return json.dumps(status, indent=1)
def _si_abbrev(self):
si = self.monitor.origin_si
if not si:
return "<LIT>"
return base32.b2a(si)[:6]
def render_title(self, ctx):
return T.title["Manifest of SI=%s" % self._si_abbrev()]
def render_header(self, ctx):
return T.p["Manifest of SI=%s" % self._si_abbrev()]
def data_items(self, ctx, data):
return self.monitor.get_status()["manifest"]
def render_row(self, ctx, path_cap):
path, cap = path_cap
ctx.fillSlots("path", self.slashify_path(path))
root = get_root(ctx)
# TODO: we need a clean consistent way to get the type of a cap string
if cap:
if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
nameurl = urllib.quote(path[-1].encode("utf-8"))
uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
nameurl)
else:
uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe=""))
ctx.fillSlots("cap", T.a(href=uri_link)[cap])
else:
ctx.fillSlots("cap", "")
return ctx.tag
class DeepSizeResults(MultiFormatPage):
# Control MultiFormatPage

View File

@ -1,28 +1,28 @@
<html xmlns:n="http://nevow.com/ns/nevow/0.1">
<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
<head>
<title n:render="title"></title>
<title t:render="title"></title>
<link href="/tahoe.css" rel="stylesheet" type="text/css"/>
<link href="/icon.png" rel="shortcut icon" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta n:render="refresh" />
<meta t:render="refresh" />
</head>
<body>
<h1><p n:render="header"></p></h1>
<h1><p t:render="header"></p></h1>
<h2 n:render="reload" />
<h2 t:render="reload" />
<table n:render="sequence" n:data="items">
<tr n:pattern="header">
<table t:render="items">
<tr t:pattern="header">
<td>Path</td>
<td>cap</td>
</tr>
<tr n:pattern="item" n:render="row">
<td><n:slot name="path"/></td>
<td><n:slot name="cap"/></td>
<tr t:render="item">
<td><t:slot name="path"/></td>
<td><t:slot name="cap"/></td>
</tr>
<tr n:pattern="empty"><td>no items in the manifest!</td></tr>
<tr t:render="empty"><td>no items in the manifest!</td></tr>
</table>