Merge pull request #665 from meejah/ticket3252-port-web-directory.manifest.2

3263 port web directory.manifest.2

src/allmydata/test/no_network.py

@@ -280,6 +280,13 @@ class NoNetworkGrid(service.MultiService):
     def __init__(self, basedir, num_clients, num_servers,
                  client_config_hooks, port_assigner):
         service.MultiService.__init__(self)
+
+        # We really need to get rid of this pattern here (and
+        # everywhere) in Tahoe where "async work" is started in
+        # __init__ For now, we at least keep the errors so they can
+        # cause tests to fail less-improperly (see _check_clients)
+        self._setup_errors = []
+
         self.port_assigner = port_assigner
         self.basedir = basedir
         fileutil.make_dirs(basedir)
@@ -300,6 +307,20 @@ class NoNetworkGrid(service.MultiService):
             d = self.make_client(i)
             d.addCallback(lambda c: self.clients.append(c))
 
+            def _bad(f):
+                self._setup_errors.append(f)
+            d.addErrback(_bad)
+
+    def _check_clients(self):
+        """
+        The anti-pattern of doing async work in __init__ means we need to
+        check if that work completed successfully. This method either
+        returns nothing or raises an exception in case __init__ failed
+        to complete properly
+        """
+        if self._setup_errors:
+            raise self._setup_errors[0].value
+
     @defer.inlineCallbacks
     def make_client(self, i, write_config=True):
         clientid = hashutil.tagged_hash("clientid", str(i))[:20]
@@ -364,6 +385,7 @@ class NoNetworkGrid(service.MultiService):
         return self.proxies_by_id.keys()
 
     def rebuild_serverlist(self):
+        self._check_clients()
         self.all_servers = frozenset(self.proxies_by_id.values())
         for c in self.clients:
             c._servers = self.all_servers
@@ -440,12 +462,14 @@ class GridTestMixin(object):
         self._record_webports_and_baseurls()
 
     def _record_webports_and_baseurls(self):
+        self.g._check_clients()
         self.client_webports = [c.getServiceNamed("webish").getPortnum()
                                 for c in self.g.clients]
         self.client_baseurls = [c.getServiceNamed("webish").getURL()
                                 for c in self.g.clients]
 
     def get_client_config(self, i=0):
+        self.g._check_clients()
         return self.g.clients[i].config
 
     def get_clientdir(self, i=0):
@@ -454,9 +478,11 @@ class GridTestMixin(object):
         return self.get_client_config(i).get_config_path()
 
     def get_client(self, i=0):
+        self.g._check_clients()
         return self.g.clients[i]
 
     def restart_client(self, i=0):
+        self.g._check_clients()
         client = self.g.clients[i]
         d = defer.succeed(None)
         d.addCallback(lambda ign: self.g.removeService(client))

src/allmydata/test/web/test_web.py

@@ -5,6 +5,8 @@ import json
 import treq
 import mock
 
+from bs4 import BeautifulSoup
+
 from twisted.application import service
 from twisted.trial import unittest
 from twisted.internet import defer
@@ -49,6 +51,10 @@ from ..common import (
     make_mutable_file_uri,
     create_mutable_filenode,
 )
+from .common import (
+    assert_soup_has_favicon,
+    assert_soup_has_text,
+)
 from allmydata.interfaces import IMutableFileNode, SDMF_VERSION, MDMF_VERSION
 from allmydata.mutable import servermap, publish, retrieve
 from .. import common_util as testutil
@@ -2038,11 +2044,12 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
             return d
         d.addCallback(getman, None)
         def _got_html(manifest):
-            self.failUnlessIn("Manifest of SI=", manifest)
+            soup = BeautifulSoup(manifest, 'html5lib')
-            self.failUnlessIn("<td>sub</td>", manifest)
+            assert_soup_has_text(self, soup, "Manifest of SI=")
-            self.failUnlessIn(self._sub_uri, manifest)
+            assert_soup_has_text(self, soup, "sub")
-            self.failUnlessIn("<td>sub/baz.txt</td>", manifest)
+            assert_soup_has_text(self, soup, self._sub_uri)
-            self.failUnlessIn(FAVICON_MARKUP, manifest)
+            assert_soup_has_text(self, soup, "sub/baz.txt")
+            assert_soup_has_favicon(self, soup)
         d.addCallback(_got_html)
 
         # both t=status and unadorned GET should be identical

src/allmydata/web/directory.py

@@ -1,12 +1,22 @@
 
 import json
 import urllib
+from datetime import timedelta
 
 from zope.interface import implementer
 from twisted.internet import defer
 from twisted.internet.interfaces import IPushProducer
 from twisted.python.failure import Failure
 from twisted.web import http
+from twisted.web.template import (
+    Element,
+    XMLFile,
+    renderElement,
+    renderer,
+    tags,
+)
+from hyperlink import URL
+from twisted.python.filepath import FilePath
 from nevow import url, rend, inevow, tags as T
 from nevow.inevow import IRequest
 
@@ -14,20 +24,40 @@ from foolscap.api import fireEventually
 
 from allmydata.util import base32
 from allmydata.util.encodingutil import to_str
-from allmydata.uri import from_string_dirnode
+from allmydata.uri import (
+    from_string_dirnode,
+    from_string,
+    CHKFileURI,
+    WriteableSSKFileURI,
+    ReadonlySSKFileURI,
+)
 from allmydata.interfaces import IDirectoryNode, IFileNode, IFilesystemNode, \
      IImmutableFileNode, IMutableFileNode, ExistingChildError, \
      NoSuchChildError, EmptyPathnameComponentError, SDMF_VERSION, MDMF_VERSION
 from allmydata.blacklist import ProhibitedNode
 from allmydata.monitor import Monitor, OperationCancelledError
 from allmydata import dirnode
-from allmydata.web.common import text_plain, WebError, \
+from allmydata.web.common import (
-     NeedOperationHandleError, \
+    text_plain,
-     boolean_of_arg, get_arg, get_root, parse_replace_arg, \
+    WebError,
-     should_create_intermediate_directories, \
+    NeedOperationHandleError,
-     getxmlfile, RenderMixin, humanize_failure, convert_children_json, \
+    boolean_of_arg,
-     get_format, get_mutable_type, get_filenode_metadata, render_time, \
+    get_arg,
-     MultiFormatPage
+    get_root,
+    parse_replace_arg,
+    should_create_intermediate_directories,
+    getxmlfile,
+    RenderMixin,
+    humanize_failure,
+    convert_children_json,
+    get_format,
+    get_mutable_type,
+    get_filenode_metadata,
+    render_time,
+    MultiFormatPage,
+    MultiFormatResource,
+    SlotsSequenceElement,
+)
 from allmydata.web.filenode import ReplaceMeMixin, \
      FileNodeHandler, PlaceHolderNodeHandler
 from allmydata.web.check_results import CheckResultsRenderer, \
@@ -943,8 +973,128 @@ class RenameForm(rend.Page):
         ctx.tag.attributes['value'] = name
         return ctx.tag
 
-class ManifestResults(MultiFormatPage, ReloadMixin):
+
-    docFactory = getxmlfile("manifest.xhtml")
+class ReloadableMonitorElement(Element):
+    """
+    Like 'ReloadMixin', but for twisted.web.template style. This
+    provides renderers for "reload" and "refesh" and a self.monitor
+    attribute (which is an instance of IMonitor)
+    """
+    refresh_time = timedelta(seconds=60)
+
+    def __init__(self, monitor):
+        self.monitor = monitor
+        super(ReloadableMonitorElement, self).__init__()
+
+    @renderer
+    def refresh(self, req, tag):
+        if self.monitor.is_finished():
+            return u""
+        tag.attributes[u"http-equiv"] = u"refresh"
+        tag.attributes[u"content"] = u"{}".format(self.refresh_time.seconds)
+        return tag
+
+    @renderer
+    def reload(self, req, tag):
+        if self.monitor.is_finished():
+            return u""
+        reload_url = URL.from_text(u"{}".format(req.path))
+        cancel_button = tags.form(
+            [
+                tags.input(type=u"submit", value=u"Cancel"),
+            ],
+            action=reload_url.replace(query={u"t": u"cancel"}).to_uri().to_text(),
+            method=u"POST",
+            enctype=u"multipart/form-data",
+        )
+
+        return tag([
+            u"Operation still running: ",
+            tags.a(
+                u"Reload",
+                href=reload_url.replace(query={u"output": u"html"}).to_uri().to_text(),
+            ),
+            cancel_button,
+        ])
+
+
+def _slashify_path(path):
+    """
+    Converts a tuple from a 'manifest' path into a string with slashes
+    in it
+    """
+    if not path:
+        return ""
+    return "/".join([p.encode("utf-8") for p in path])
+
+
+def _cap_to_link(root, path, cap):
+    """
+    Turns a capability-string into a WebAPI link tag
+
+    :param text root: the root piece of the URI
+
+    :param text cap: the capability-string
+
+    :returns: something suitable for `IRenderable`, specifically
+        either a valid local link (tags.a instance) to the capability
+        or an empty string.
+    """
+    if cap:
+        root_url = URL.from_text(u"{}".format(root))
+        cap_obj = from_string(cap)
+        if isinstance(cap_obj, (CHKFileURI, WriteableSSKFileURI, ReadonlySSKFileURI)):
+            uri_link = root_url.child(
+                u"file",
+                u"{}".format(urllib.quote(cap)),
+                u"{}".format(urllib.quote(path[-1])),
+            )
+        else:
+            uri_link = root_url.child(
+                u"uri",
+                u"{}".format(urllib.quote(cap, safe="")),
+            )
+        return tags.a(cap, href=uri_link.to_text())
+    else:
+        return u""
+
+
+class ManifestElement(ReloadableMonitorElement):
+    loader = XMLFile(FilePath(__file__).sibling("manifest.xhtml"))
+
+    def _si_abbrev(self):
+        si = self.monitor.origin_si
+        if not si:
+            return "<LIT>"
+        return base32.b2a(si)[:6]
+
+    @renderer
+    def title(self, req, tag):
+        return tag(
+            "Manifest of SI={}".format(self._si_abbrev())
+        )
+
+    @renderer
+    def header(self, req, tag):
+        return tag(
+            "Manifest of SI={}".format(self._si_abbrev())
+        )
+
+    @renderer
+    def items(self, req, tag):
+        manifest = self.monitor.get_status()["manifest"]
+        root = get_root(req)
+        rows = [
+            {
+                "path": _slashify_path(path),
+                "cap": _cap_to_link(root, path, cap),
+            }
+            for path, cap in manifest
+        ]
+        return SlotsSequenceElement(tag, rows)
+
+
+class ManifestResults(MultiFormatResource, ReloadMixin):
 
     # Control MultiFormatPage
     formatArgument = "output"
@@ -954,21 +1104,19 @@ class ManifestResults(MultiFormatPage, ReloadMixin):
         self.client = client
         self.monitor = monitor
 
-    # The default format is HTML but the HTML renderer is just renderHTTP.
+    def render_HTML(self, req):
-    render_HTML = None
+        return renderElement(
-
+            req,
-    def slashify_path(self, path):
+            ManifestElement(self.monitor)
-        if not path:
+        )
-            return ""
-        return "/".join([p.encode("utf-8") for p in path])
 
     def render_TEXT(self, req):
         req.setHeader("content-type", "text/plain")
         lines = []
         is_finished = self.monitor.is_finished()
         lines.append("finished: " + {True: "yes", False: "no"}[is_finished])
-        for (path, cap) in self.monitor.get_status()["manifest"]:
+        for path, cap in self.monitor.get_status()["manifest"]:
-            lines.append(self.slashify_path(path) + " " + cap)
+            lines.append(_slashify_path(path) + " " + cap)
         return "\n".join(lines) + "\n"
 
     def render_JSON(self, req):
@@ -1002,37 +1150,6 @@ class ManifestResults(MultiFormatPage, ReloadMixin):
             # CPU.
         return json.dumps(status, indent=1)
 
-    def _si_abbrev(self):
-        si = self.monitor.origin_si
-        if not si:
-            return "<LIT>"
-        return base32.b2a(si)[:6]
-
-    def render_title(self, ctx):
-        return T.title["Manifest of SI=%s" % self._si_abbrev()]
-
-    def render_header(self, ctx):
-        return T.p["Manifest of SI=%s" % self._si_abbrev()]
-
-    def data_items(self, ctx, data):
-        return self.monitor.get_status()["manifest"]
-
-    def render_row(self, ctx, path_cap):
-        path, cap = path_cap
-        ctx.fillSlots("path", self.slashify_path(path))
-        root = get_root(ctx)
-        # TODO: we need a clean consistent way to get the type of a cap string
-        if cap:
-            if cap.startswith("URI:CHK") or cap.startswith("URI:SSK"):
-                nameurl = urllib.quote(path[-1].encode("utf-8"))
-                uri_link = "%s/file/%s/@@named=/%s" % (root, urllib.quote(cap),
-                                                       nameurl)
-            else:
-                uri_link = "%s/uri/%s" % (root, urllib.quote(cap, safe=""))
-            ctx.fillSlots("cap", T.a(href=uri_link)[cap])
-        else:
-            ctx.fillSlots("cap", "")
-        return ctx.tag
 
 class DeepSizeResults(MultiFormatPage):
     # Control MultiFormatPage

src/allmydata/web/manifest.xhtml

@@ -1,28 +1,28 @@
-<html xmlns:n="http://nevow.com/ns/nevow/0.1">
+<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1">
   <head>
-    <title n:render="title"></title>
+    <title t:render="title"></title>
     <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
     <link href="/icon.png" rel="shortcut icon" />
     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-    <meta n:render="refresh" />
+    <meta t:render="refresh" />
   </head>
   <body>
 
-<h1><p n:render="header"></p></h1>
+<h1><p t:render="header"></p></h1>
 
-<h2 n:render="reload" />
+<h2 t:render="reload" />
 
-<table n:render="sequence" n:data="items">
+<table t:render="items">
-  <tr n:pattern="header">
+  <tr t:pattern="header">
     <td>Path</td>
     <td>cap</td>
   </tr>
-  <tr n:pattern="item" n:render="row">
+  <tr t:render="item">
-    <td><n:slot name="path"/></td>
+    <td><t:slot name="path"/></td>
-    <td><n:slot name="cap"/></td>
+    <td><t:slot name="cap"/></td>
   </tr>
 
-  <tr n:pattern="empty"><td>no items in the manifest!</td></tr>
+  <tr t:render="empty"><td>no items in the manifest!</td></tr>
 
 </table>