Use beautifulsoup4+html5lib for web tests.

Assertions about the rendered template will be easier to write and
don't rely on perfect substring matches.

setup.py

@@ -341,6 +341,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "towncrier",
               "testtools",
               "fixtures",
+              "beautifulsoup4",
+              "html5lib",
           ],
           "tor": [
               "foolscap[tor] >= 0.12.5",

src/allmydata/test/web/common.py

@@ -1,6 +1,18 @@
 
+import re
+
 unknown_rwcap = u"lafs://from_the_future_rw_\u263A".encode('utf-8')
 unknown_rocap = u"ro.lafs://readonly_from_the_future_ro_\u263A".encode('utf-8')
 unknown_immcap = u"imm.lafs://immutable_from_the_future_imm_\u263A".encode('utf-8')
 
 FAVICON_MARKUP = '<link href="/icon.png" rel="shortcut icon" />'
+
+def assert_soup_has_favicon(testcase, soup):
+    links = soup.find_all('link', rel='shortcut icon')
+    testcase.assert_(
+        any(t['href'] == '/icon.png' for t in links), soup)
+
+def assert_soup_has_text(testcase, soup, text):
+    testcase.assert_(
+        soup.find_all(string=re.compile(re.escape(text))),
+        soup)

src/allmydata/test/web/test_introducer.py

@@ -1,3 +1,5 @@
+import re
+from bs4 import BeautifulSoup
 from os.path import join
 from twisted.trial import unittest
 from twisted.internet import reactor
@@ -6,7 +8,8 @@ from twisted.internet import defer
 from allmydata.introducer import create_introducer
 from allmydata import node
 from .common import (
-    FAVICON_MARKUP,
+    assert_soup_has_favicon,
+    assert_soup_has_text,
 )
 from ..common import (
     SameProcessStreamEndpointAssigner,
@@ -47,7 +50,8 @@ class IntroducerWeb(unittest.TestCase):
 
         url = "http://localhost:%d/" % self.ws.getPortnum()
         res = yield do_http("get", url)
-        self.failUnlessIn('Welcome to the Tahoe-LAFS Introducer', res)
-        self.failUnlessIn(FAVICON_MARKUP, res)
-        self.failUnlessIn('Page rendered at', res)
-        self.failUnlessIn('Tahoe-LAFS code imported from:', res)
+        soup = BeautifulSoup(res, 'html5lib')
+        assert_soup_has_text(self, soup, 'Welcome to the Tahoe-LAFS Introducer')
+        assert_soup_has_favicon(self, soup)
+        assert_soup_has_text(self, soup, 'Page rendered at')
+        assert_soup_has_text(self, soup, 'Tahoe-LAFS code imported from:')