Merge pull request #649 from habnabit/move-to-twt--introducer

Port introweb to use twisted.web.template

Fixes: ticket:3245

setup.py

@@ -358,6 +358,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "towncrier",
               "testtools",
               "fixtures",
+              "beautifulsoup4",
+              "html5lib",
           ] + tor_requires + i2p_requires,
           "tor": tor_requires,
           "i2p": i2p_requires,

src/allmydata/test/test_introducer.py

@@ -11,6 +11,7 @@ from testtools.matchers import (
 from twisted.internet import defer, address
 from twisted.python import log
 from twisted.python.filepath import FilePath
+from twisted.web.template import flattenString
 
 from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue
 from twisted.application import service
@@ -592,7 +593,12 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
             # now check the web status, make sure it renders without error
             ir = introweb.IntroducerRoot(self.parent)
             self.parent.nodeid = "NODEID"
-            text = ir.renderSynchronously().decode("utf-8")
+            log.msg("_check1 done")
+            return flattenString(None, ir._create_element())
+        d.addCallback(_check1)
+
+        def _check2(flattened_bytes):
+            text = flattened_bytes.decode("utf-8")
             self.assertIn(NICKNAME % "0", text) # a v2 client
             self.assertIn(NICKNAME % "1", text) # another v2 client
             for i in range(NUM_STORAGE):
@@ -601,8 +607,8 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
                 # make sure there isn't a double-base32ed string too
                 self.assertNotIn(idlib.nodeid_b2a(printable_serverids[i]), text,
                               (i,printable_serverids[i],text))
-            log.msg("_check1 done")
+            log.msg("_check2 done")
-        d.addCallback(_check1)
+        d.addCallback(_check2)
 
         # force an introducer reconnect, by shutting down the Tub it's using
         # and starting a new Tub (with the old introducer). Everybody should

src/allmydata/test/web/common.py

@@ -1,6 +1,29 @@
 
+import re
+
 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
 
 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
+
+
+def assert_soup_has_favicon(testcase, soup):
+    """
+    Using a ``TestCase`` object ``testcase``, assert that the passed in
+    ``BeautifulSoup`` object ``soup`` contains the tahoe favicon link.
+    """
+    links = soup.find_all(u'link', rel=u'shortcut icon')
+    testcase.assert_(
+        any(t[u'href'] == u'/icon.png' for t in links), soup)
+
+
+def assert_soup_has_text(testcase, soup, text):
+    """
+    Using a ``TestCase`` object ``testcase``, assert that the passed in
+    ``BeautifulSoup`` object ``soup`` contains the passed in ``text`` anywhere
+    as a text node.
+    """
+    testcase.assert_(
+        soup.find_all(string=re.compile(re.escape(text))),
+        soup)

src/allmydata/test/web/test_introducer.py

@@ -1,3 +1,4 @@
+from bs4 import BeautifulSoup
 from os.path import join
 from twisted.trial import unittest
 from twisted.internet import reactor
@@ -6,13 +7,15 @@ from twisted.internet import defer
 from allmydata.introducer import create_introducer
 from allmydata import node
 from .common import (
-    FAVICON_MARKUP,
+    assert_soup_has_favicon,
+    assert_soup_has_text,
 )
 from ..common import (
     SameProcessStreamEndpointAssigner,
 )
 from ..common_web import do_http
 
+
 class IntroducerWeb(unittest.TestCase):
     def setUp(self):
         self.node = None
@@ -47,7 +50,8 @@ class IntroducerWeb(unittest.TestCase):
 
         url = "http://localhost:%d/" % self.ws.getPortnum()
         res = yield do_http("get", url)
-        self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
+        soup = BeautifulSoup(res, 'html5lib')
-        self.failUnlessIn(FAVICON_MARKUP, res)
+        assert_soup_has_text(self, soup, u'Welcome to the Tahoe-LAFS Introducer')
-        self.failUnlessIn('Page rendered at', res)
+        assert_soup_has_favicon(self, soup)
-        self.failUnlessIn('Tahoe-LAFS code imported from:', res)
+        assert_soup_has_text(self, soup, u'Page rendered at')
+        assert_soup_has_text(self, soup, u'Tahoe-LAFS code imported from:')

src/allmydata/web/common.py

@@ -2,7 +2,7 @@
 import time
 import json
 
-from twisted.web import http, server, resource
+from twisted.web import http, server, resource, template
 from twisted.python import log
 from twisted.python.failure import Failure
 from zope.interface import Interface
@@ -460,6 +460,102 @@ class MultiFormatPage(Page):
             return lambda ctx: renderer(IRequest(ctx))
 
 
+class MultiFormatResource(resource.Resource, object):
+    """
+    ``MultiFormatResource`` is a ``resource.Resource`` that can be rendered in
+    a number of different formats.
+
+    Rendered format is controlled by a query argument (given by
+    ``self.formatArgument``).  Different resources may support different
+    formats but ``json`` is a pretty common one.  ``html`` is the default
+    format if nothing else is given as the ``formatDefault``.
+    """
+    formatArgument = "t"
+    formatDefault = None
+
+    def render(self, req):
+        """
+        Dispatch to a renderer for a particular format, as selected by a query
+        argument.
+
+        A renderer for the format given by the query argument matching
+        ``formatArgument`` will be selected and invoked.  render_HTML will be
+        used as a default if no format is selected (either by query arguments
+        or by ``formatDefault``).
+
+        :return: The result of the selected renderer.
+        """
+        t = get_arg(req, self.formatArgument, self.formatDefault)
+        renderer = self._get_renderer(t)
+        return renderer(req)
+
+    def _get_renderer(self, fmt):
+        """
+        Get the renderer for the indicated format.
+
+        :param str fmt: The format.  If a method with a prefix of ``render_``
+            and a suffix of this format (upper-cased) is found, it will be
+            used.
+
+        :return: A callable which takes a twisted.web Request and renders a
+            response.
+        """
+        renderer = None
+
+        if fmt is not None:
+            try:
+                renderer = getattr(self, "render_{}".format(fmt.upper()))
+            except AttributeError:
+                raise WebError(
+                    "Unknown {} value: {!r}".format(self.formatArgument, fmt),
+                )
+
+        if renderer is None:
+            renderer = self.render_HTML
+
+        return renderer
+
+
+class SlotsSequenceElement(template.Element):
+    """
+    ``SlotsSequenceElement` is a minimal port of nevow's sequence renderer for
+    twisted.web.template.
+
+    Tags passed in to be templated will have two renderers available: ``item``
+    and ``tag``.
+    """
+
+    def __init__(self, tag, seq):
+        self.loader = template.TagLoader(tag)
+        self.seq = seq
+
+    @template.renderer
+    def item(self, request, tag):
+        """
+        A template renderer for each sequence item.
+
+        ``tag`` will be cloned for each item in the sequence provided, and its
+        slots filled from the sequence item. Each item must be dict-like enough
+        for ``tag.fillSlots(**item)``. Each cloned tag will be siblings with no
+        separator beween them.
+        """
+        for item in self.seq:
+            yield tag.clone(deep=False).fillSlots(**item)
+
+    @template.renderer
+    def empty(self, request, tag):
+        """
+        A template renderer for empty sequences.
+
+        This renderer will either return ``tag`` unmodified if the provided
+        sequence has no items, or return the empty string if there are any
+        items.
+        """
+        if len(self.seq) > 0:
+            return u''
+        else:
+            return tag
+
 
 class TokenOnlyWebApi(resource.Resource, object):
     """

src/allmydata/web/introducer.xhtml

@@ -1,4 +1,4 @@
-<html xmlns:n="http://nevow.com/ns/nevow/0.1"><head>
+<html xmlns:t="http://twistedmatrix.com/ns/twisted.web.template/0.1"><head>
   <title>Tahoe-LAFS - Introducer Status</title>
   <link href="/tahoe.css" rel="stylesheet" type="text/css"/>
   <link href="/icon.png" rel="shortcut icon" />
@@ -10,23 +10,23 @@
 <div class="section" id="this-client">
   <h2>This Introducer</h2>
 
-  <table class="node-info table-headings-left">
+  <table class="node-info table-headings-left" t:render="node_data">
-    <tr><th>My nodeid:</th> <td class="nodeid mine data-chars" n:render="string" n:data="my_nodeid" /></tr>
+    <tr><th>My nodeid:</th> <td class="nodeid mine data-chars"><t:slot name="my_nodeid" /></td></tr>
-    <tr><th>My versions:</th> <td n:render="string" n:data="version" /></tr>
+    <tr><th>My versions:</th> <td><t:slot name="version" /></td></tr>
-    <tr><th>Tahoe-LAFS code imported from:</th> <td n:render="data" n:data="import_path" /></tr>
+    <tr><th>Tahoe-LAFS code imported from:</th> <td><t:slot name="import_path" /></td></tr>
   </table>
 </div>
 
-<div>Announcement Summary: <span n:render="announcement_summary" /></div>
+<div>Announcement Summary: <span t:render="announcement_summary" /></div>
-<div>Subscription Summary: <span n:render="client_summary" /></div>
+<div>Subscription Summary: <span t:render="client_summary" /></div>
 
 <br />
 
 
 <div class="section">
 <h2>Service Announcements</h2>
-<table class="services table-headings-top" n:render="sequence" n:data="services">
+<table class="services table-headings-top" t:render="services">
-  <tr n:pattern="header">
+  <tr>
     <th class="nickname-and-peerid">
       <div class="service-nickname">Nickname</div>
       <div class="nodeid data-chars">ServerID</div></th>
@@ -34,23 +34,23 @@
     <th>Version</th>
     <th>Service Name</th>
   </tr>
-  <tr n:pattern="item" n:render="service_row">
+  <tr t:render="item">
     <td class="nickname-and-peerid">
-      <div class="nickname"><n:slot name="nickname"/></div>
+      <div class="nickname"><t:slot name="nickname"/></div>
-      <div class="nodeid data-chars"><n:slot name="serverid"/></div></td>
+      <div class="nodeid data-chars"><t:slot name="serverid"/></div></td>
-    <td class="service-announced"><n:attr name="title"><n:slot name="connection-hints"/></n:attr><n:slot name="announced"/></td>
+    <td class="service-announced"><t:attr name="title"><t:slot name="connection-hints"/></t:attr><t:slot name="announced"/></td>
-    <td class="service-version"><n:slot name="version"/></td>
+    <td class="service-version"><t:slot name="version"/></td>
-    <td class="service-service-name"><n:slot name="service_name"/></td>
+    <td class="service-service-name"><t:slot name="service_name"/></td>
   </tr>
-  <tr n:pattern="empty"><td>no peers!</td></tr>
+  <tr t:render="empty"><td>no peers!</td></tr>
 </table>
 </div>
 
 
 <div>
 <h2>Subscribed Clients</h2>
-<table class="services table-headings-top" n:render="sequence" n:data="subscribers">
+<table class="services table-headings-top" t:render="subscribers">
-  <tr n:pattern="header">
+  <tr>
     <th class="nickname-and-peerid">
       <div class="service-nickname">Nickname</div>
       <div class="nodeid data-chars">Tub ID</div></th>
@@ -59,20 +59,20 @@
     <th>Version</th>
     <th>Subscribed To</th>
   </tr>
-  <tr n:pattern="item" n:render="subscriber_row">
+  <tr t:render="item">
     <td class="nickname-and-peerid">
-      <div class="nickname"><n:slot name="nickname"/></div>
+      <div class="nickname"><t:slot name="nickname"/></div>
-      <div class="nodeid data-chars"><n:slot name="tubid"/></div></td>
+      <div class="nodeid data-chars"><t:slot name="tubid"/></div></td>
-    <td><n:slot name="connected"/></td>
+    <td><t:slot name="connected"/></td>
-    <td class="service-since"><n:slot name="since"/></td>
+    <td class="service-since"><t:slot name="since"/></td>
-    <td class="service-version"><n:slot name="version"/></td>
+    <td class="service-version"><t:slot name="version"/></td>
-    <td class="service-service-name"><n:slot name="service_name"/></td>
+    <td class="service-service-name"><t:slot name="service_name"/></td>
   </tr>
-  <tr n:pattern="empty"><td>no subscribers!</td></tr>
+  <tr t:render="empty"><td>no subscribers!</td></tr>
 </table>
 </div>
 
-<p class="minutia">Page rendered at <span n:render="data" n:data="rendered_at" /></p>
+<p class="minutia" t:render="node_data">Page rendered at <span><t:slot name="rendered_at" /></span></p>
 
   </body>
 </html>

src/allmydata/web/introweb.py

@@ -1,35 +1,56 @@
 
 import time, os
-from nevow import rend
+from pkg_resources import resource_filename
-from nevow.static import File as nevow_File
+from twisted.web.template import Element, XMLFile, renderElement, renderer
-from nevow.util import resource_filename
+from twisted.python.filepath import FilePath
+from twisted.web import static
 import allmydata
 import json
 from allmydata.version_checks import get_package_versions_string
 from allmydata.util import idlib
 from allmydata.web.common import (
-    getxmlfile,
     render_time,
-    MultiFormatPage,
+    MultiFormatResource,
+    SlotsSequenceElement,
 )
 
 
-class IntroducerRoot(MultiFormatPage):
+class IntroducerRoot(MultiFormatResource):
+    """
+    A ``Resource`` intended as the root resource for introducers.
 
-    addSlash = True
+    :param _IntroducerNode introducer_node: The introducer node to template
-    docFactory = getxmlfile("introducer.xhtml")
+        information about.
-
+    """
-    child_operations = None
 
     def __init__(self, introducer_node):
+        super(IntroducerRoot, self).__init__()
         self.introducer_node = introducer_node
         self.introducer_service = introducer_node.getServiceNamed("introducer")
-        rend.Page.__init__(self, introducer_node)
+        # necessary as a root Resource
+        self.putChild("", self)
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
-            self.putChild(filen, nevow_File(os.path.join(static_dir, filen)))
+            self.putChild(filen, static.File(os.path.join(static_dir, filen)))
+
+    def _create_element(self):
+        """
+        Create a ``IntroducerRootElement`` which can be flattened into an HTML
+        response.
+        """
+        return IntroducerRootElement(
+            self.introducer_node, self.introducer_service)
+
+    def render_HTML(self, req):
+        """
+        Render an HTML template describing this introducer node.
+        """
+        return renderElement(req, self._create_element())
 
     def render_JSON(self, req):
+        """
+        Render JSON describing this introducer node.
+        """
         res = {}
 
         counts = {}
@@ -37,7 +58,7 @@ class IntroducerRoot(MultiFormatPage):
             if s.service_name not in counts:
                 counts[s.service_name] = 0
             counts[s.service_name] += 1
-        res["subscription_summary"] = counts
+        res[u"subscription_summary"] = counts
 
         announcement_summary = {}
         for ad in self.introducer_service.get_announcements():
@@ -45,21 +66,40 @@ class IntroducerRoot(MultiFormatPage):
             if service_name not in announcement_summary:
                 announcement_summary[service_name] = 0
             announcement_summary[service_name] += 1
-        res["announcement_summary"] = announcement_summary
+        res[u"announcement_summary"] = announcement_summary
 
-        return json.dumps(res, indent=1) + "\n"
+        return json.dumps(res, indent=1) + b"\n"
 
-    # FIXME: This code is duplicated in root.py and introweb.py.
-    def data_rendered_at(self, ctx, data):
-        return render_time(time.time())
-    def data_version(self, ctx, data):
-        return get_package_versions_string()
-    def data_import_path(self, ctx, data):
-        return str(allmydata).replace("/", "/ ") # XXX kludge for wrapping
-    def data_my_nodeid(self, ctx, data):
-        return idlib.nodeid_b2a(self.introducer_node.nodeid)
 
-    def render_announcement_summary(self, ctx, data):
+class IntroducerRootElement(Element):
+    """
+    An ``Element`` HTML template which can be flattened to describe this
+    introducer node.
+
+    :param _IntroducerNode introducer_node: The introducer node to describe.
+    :param IntroducerService introducer_service: The introducer service created
+        by the node.
+    """
+
+    loader = XMLFile(FilePath(__file__).sibling("introducer.xhtml"))
+
+    def __init__(self, introducer_node, introducer_service):
+        super(IntroducerRootElement, self).__init__()
+        self.introducer_node = introducer_node
+        self.introducer_service = introducer_service
+        self.node_data_dict = {
+            "my_nodeid": idlib.nodeid_b2a(self.introducer_node.nodeid),
+            "version": get_package_versions_string(),
+            "import_path": str(allmydata).replace("/", "/ "),  # XXX kludge for wrapping
+            "rendered_at": render_time(time.time()),
+        }
+
+    @renderer
+    def node_data(self, req, tag):
+        return tag.fillSlots(**self.node_data_dict)
+
+    @renderer
+    def announcement_summary(self, req, tag):
         services = {}
         for ad in self.introducer_service.get_announcements():
             if ad.service_name not in services:
@@ -67,44 +107,43 @@ class IntroducerRoot(MultiFormatPage):
             services[ad.service_name] += 1
         service_names = services.keys()
         service_names.sort()
-        return ", ".join(["%s: %d" % (service_name, services[service_name])
+        return u", ".join(u"{}: {}".format(service_name, services[service_name])
-                          for service_name in service_names])
+                          for service_name in service_names)
 
-    def render_client_summary(self, ctx, data):
+    @renderer
+    def client_summary(self, req, tag):
         counts = {}
         for s in self.introducer_service.get_subscribers():
             if s.service_name not in counts:
                 counts[s.service_name] = 0
             counts[s.service_name] += 1
-        return ", ".join([ "%s: %d" % (name, counts[name])
+        return u", ".join(u"{}: {}".format(name, counts[name])
-                           for name in sorted(counts.keys()) ] )
+                          for name in sorted(counts.keys()))
 
-    def data_services(self, ctx, data):
+    @renderer
+    def services(self, req, tag):
         services = self.introducer_service.get_announcements()
         services.sort(key=lambda ad: (ad.service_name, ad.nickname))
-        return services
+        services = [{
+            "serverid": ad.serverid,
+            "nickname": ad.nickname,
+            "connection-hints":
+                u"connection hints: " + u" ".join(ad.connection_hints),
+            "connected": u"?",
+            "announced": render_time(ad.when),
+            "version": ad.version,
+            "service_name": ad.service_name,
+        } for ad in services]
+        return SlotsSequenceElement(tag, services)
 
-    def render_service_row(self, ctx, ad):
+    @renderer
-        ctx.fillSlots("serverid", ad.serverid)
+    def subscribers(self, req, tag):
-        ctx.fillSlots("nickname", ad.nickname)
+        subscribers = [{
-        ctx.fillSlots("connection-hints",
+            "nickname": s.nickname,
-                      "connection hints: " + " ".join(ad.connection_hints))
+            "tubid": s.tubid,
-        ctx.fillSlots("connected", "?")
+            "connected": s.remote_address,
-        when_s = render_time(ad.when)
+            "since": render_time(s.when),
-        ctx.fillSlots("announced", when_s)
+            "version": s.version,
-        ctx.fillSlots("version", ad.version)
+            "service_name": s.service_name,
-        ctx.fillSlots("service_name", ad.service_name)
+        } for s in self.introducer_service.get_subscribers()]
-        return ctx.tag
+        return SlotsSequenceElement(tag, subscribers)
-
-    def data_subscribers(self, ctx, data):
-        return self.introducer_service.get_subscribers()
-
-    def render_subscriber_row(self, ctx, s):
-        ctx.fillSlots("nickname", s.nickname)
-        ctx.fillSlots("tubid", s.tubid)
-        ctx.fillSlots("connected", s.remote_address)
-        since_s = render_time(s.when)
-        ctx.fillSlots("since", since_s)
-        ctx.fillSlots("version", s.version)
-        ctx.fillSlots("service_name", s.service_name)
-        return ctx.tag