From e981fea00790765af81422d1c02dba0cfd4295e1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Sep 2020 16:24:25 -0400 Subject: [PATCH 1/8] make FakeCanary more realistic --- src/allmydata/test/common_py3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 5b791fd0a..c3a84189b 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -159,9 +159,12 @@ class FakeCanary(object): if self.ignore: return del self.disconnectors[marker] + def getRemoteTubID(self): + return None + def getPeer(self): + return "" class LoggingServiceParent(service.MultiService): def log(self, *args, **kwargs): return log.msg(*args, **kwargs) - From 16ab1690deced7e7a4a639f642d9c366f803e168 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Sep 2020 16:26:20 -0400 Subject: [PATCH 2/8] rewrite _test_introweb part of SystemTest as its own test suite --- src/allmydata/test/test_system.py | 54 ------ src/allmydata/test/web/test_introducer.py | 202 +++++++++++++++++++--- 2 files changed, 177 insertions(+), 79 deletions(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index cc1ee21bb..21da0a914 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -11,7 +11,6 @@ from twisted.internet import defer from twisted.internet.defer import inlineCallbacks from twisted.application import service -import allmydata from allmydata import client, uri from allmydata.introducer.server import create_introducer from allmydata.storage.mutable import MutableShareFile @@ -1629,7 +1628,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): for c in self.clients: c.encoding_params['happy'] = 1 d.addCallback(_new_happy_semantics) - d.addCallback(self._test_introweb) d.addCallback(self.log, "starting publish") d.addCallback(self._do_publish1) d.addCallback(self._test_runner) @@ -1668,58 +1666,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): d.addCallback(self._test_checker) return d - def _test_introweb(self, res): - d = do_http("get", self.introweb_url) - def _check(res): - try: - self.failUnless("%s: %s" % (allmydata.__appname__, allmydata.__version__) in res) - verstr = str(allmydata.__version__) - - # The Python "rational version numbering" convention - # disallows "-r$REV" but allows ".post$REV" - # instead. Eventually we'll probably move to - # that. When we do, this test won't go red: - ix = verstr.rfind('-r') - if ix != -1: - altverstr = verstr[:ix] + '.post' + verstr[ix+2:] - else: - ix = verstr.rfind('.post') - if ix != -1: - altverstr = verstr[:ix] + '-r' + verstr[ix+5:] - else: - altverstr = verstr - - appverstr = "%s: %s" % (allmydata.__appname__, verstr) - newappverstr = "%s: %s" % (allmydata.__appname__, altverstr) - - self.failUnless((appverstr in res) or (newappverstr in res), (appverstr, newappverstr, res)) - self.failUnless("Announcement Summary: storage: 5" in res) - self.failUnless("Subscription Summary: storage: 5" in res) - self.failUnless("tahoe.css" in res) - except unittest.FailTest: - print() - print("GET %s output was:" % self.introweb_url) - print(res) - raise - d.addCallback(_check) - # make sure it serves the CSS too - d.addCallback(lambda res: do_http("get", self.introweb_url+"tahoe.css")) - d.addCallback(lambda res: do_http("get", self.introweb_url + "?t=json")) - def _check_json(res): - data = json.loads(res) - try: - self.failUnlessEqual(data["subscription_summary"], - {"storage": 5}) - self.failUnlessEqual(data["announcement_summary"], - {"storage": 5}) - except unittest.FailTest: - print() - print("GET %s?t=json output was:" % self.introweb_url) - print(res) - raise - d.addCallback(_check_json) - return d - def _do_publish1(self, res): ut = upload.Data(self.data, convergence=None) c0 = self.clients[0] diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 55db61a13..fad683be7 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,19 +1,66 @@ -from bs4 import BeautifulSoup +import json from os.path import join +from os import makedirs + +from bs4 import BeautifulSoup + from twisted.trial import unittest from twisted.internet import reactor -from foolscap.api import fireEventually, flushEventualQueue from twisted.internet import defer -from allmydata.introducer import create_introducer + +from foolscap.api import ( + fireEventually, + flushEventualQueue, + Tub, +) + +import allmydata +from allmydata.introducer import ( + create_introducer, +) +from allmydata.introducer.server import ( + _IntroducerNode, +) +from allmydata.web.introweb import ( + IntroducerRoot, +) + from allmydata import node from .common import ( assert_soup_has_favicon, assert_soup_has_text, + assert_soup_has_tag_with_attributes, ) from ..common import ( SameProcessStreamEndpointAssigner, ) -from ..common_web import do_http +from ..common_py3 import ( + FakeCanary, +) +from ..common_web import ( + do_http, + render, +) + + +@defer.inlineCallbacks +def create_introducer_webish(reactor, port_assigner, basedir): + node.create_node_dir(basedir, "testing") + _, port_endpoint = port_assigner.assign(reactor) + with open(join(basedir, "tahoe.cfg"), "w") as f: + f.write( + "[node]\n" + "tub.location = 127.0.0.1:1\n" + + "web.port = {}\n".format(port_endpoint) + ) + + intro_node = yield create_introducer(basedir) + ws = intro_node.getServiceNamed("webish") + + yield fireEventually(None) + intro_node.startService() + + defer.returnValue((intro_node, ws)) class IntroducerWeb(unittest.TestCase): @@ -24,34 +71,139 @@ class IntroducerWeb(unittest.TestCase): self.addCleanup(self.port_assigner.tearDown) def tearDown(self): - d = defer.succeed(None) - if self.node: - d.addCallback(lambda ign: self.node.stopService()) - d.addCallback(flushEventualQueue) - return d + return flushEventualQueue(None) @defer.inlineCallbacks def test_welcome(self): - basedir = self.mktemp() - node.create_node_dir(basedir, "testing") - _, port_endpoint = self.port_assigner.assign(reactor) - with open(join(basedir, "tahoe.cfg"), "w") as f: - f.write( - "[node]\n" - "tub.location = 127.0.0.1:1\n" + - "web.port = {}\n".format(port_endpoint) - ) + node, ws = yield create_introducer_webish( + reactor, + self.port_assigner, + self.mktemp(), + ) + self.addCleanup(node.stopService) - self.node = yield create_introducer(basedir) - self.ws = self.node.getServiceNamed("webish") - - yield fireEventually(None) - self.node.startService() - - url = "http://localhost:%d/" % self.ws.getPortnum() + url = "http://localhost:%d/" % (ws.getPortnum(),) res = yield do_http("get", url) soup = BeautifulSoup(res, 'html5lib') assert_soup_has_text(self, soup, u'Welcome to the Tahoe-LAFS Introducer') assert_soup_has_favicon(self, soup) assert_soup_has_text(self, soup, u'Page rendered at') assert_soup_has_text(self, soup, u'Tahoe-LAFS code imported from:') + + @defer.inlineCallbacks + def test_basic_information(self): + """ + The introducer web page includes the software version and several other + simple pieces of information. + """ + node, ws = yield create_introducer_webish( + reactor, + self.port_assigner, + self.mktemp(), + ) + self.addCleanup(node.stopService) + + url = "http://localhost:%d/" % (ws.getPortnum(),) + res = yield do_http("get", url) + soup = BeautifulSoup(res, 'html5lib') + res = yield do_http("get", url) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_text( + self, + soup, + u"%s: %s" % (allmydata.__appname__, allmydata.__version__), + ) + assert_soup_has_text(self, soup, u"no peers!") + assert_soup_has_text(self, soup, u"subscribers!") + assert_soup_has_tag_with_attributes( + self, + soup, + "link", + {"href": "/tahoe.css"}, + ) + + @defer.inlineCallbacks + def test_tahoe_css(self): + """ + The introducer serves the css. + """ + node, ws = yield create_introducer_webish( + reactor, + self.port_assigner, + self.mktemp(), + ) + self.addCleanup(node.stopService) + + url = "http://localhost:%d/tahoe.css" % (ws.getPortnum(),) + + # Just don't return an error. If it does, do_http will raise + # something. + yield do_http("get", url) + + @defer.inlineCallbacks + def test_json_front_page(self): + """ + The front page can be served as json. + """ + node, ws = yield create_introducer_webish( + reactor, + self.port_assigner, + self.mktemp(), + ) + self.addCleanup(node.stopService) + + url = "http://localhost:%d/?t=json" % (ws.getPortnum(),) + res = yield do_http("get", url) + data = json.loads(res) + self.assertEqual(data["subscription_summary"], {}) + self.assertEqual(data["announcement_summary"], {}) + + +class IntroducerRootTests(unittest.TestCase): + """ + Tests for ``IntroducerRoot``. + """ + def test_json(self): + """ + The JSON response includes totals for the number of subscriptions and + announcements of each service type. + """ + config = node.config_from_string(self.mktemp(), "", "") + config.get_private_path = lambda ignored: self.mktemp() + main_tub = Tub() + main_tub.listenOn(b"tcp:0") + main_tub.setLocation(b"tcp:127.0.0.1:1") + introducer_node = _IntroducerNode(config, main_tub, None, None, None) + + introducer_service = introducer_node.getServiceNamed("introducer") + for n in range(2): + introducer_service.add_subscriber( + FakeCanary(), + "arbitrary", + {"info": "info"}, + ) + + # It would be nice to use the publish method but then we have to + # generate a correctly signed message which I don't feel like doing. + ann_t = ("msg", "sig", "key") + ann = {"service-name": "arbitrary"} + introducer_service._announcements[("arbitrary", "key")] = ( + ann_t, + FakeCanary(), + ann, + 0, + ) + + resource = IntroducerRoot(introducer_node) + response = json.loads( + self.successResultOf( + render(resource, t=[b"json"]), + ), + ) + self.assertEqual( + response, + { + u"subscription_summary": {"arbitrary": 2}, + u"announcement_summary": {"arbitrary": 1}, + }, + ) From 1c5f4e2bf07079fdbbe23535f3ff3963db46d358 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 Sep 2020 18:36:17 -0400 Subject: [PATCH 3/8] Compatible with the interface I used in test_web.py --- src/allmydata/test/web/test_introducer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index fad683be7..b752022df 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -197,7 +197,7 @@ class IntroducerRootTests(unittest.TestCase): resource = IntroducerRoot(introducer_node) response = json.loads( self.successResultOf( - render(resource, t=[b"json"]), + render(resource, {"t": [b"json"]}), ), ) self.assertEqual( From f13390d3d400402e4c87a9c0357a31a12944af9c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 25 Sep 2020 09:26:56 -0400 Subject: [PATCH 4/8] news fragment --- newsfragments/3438.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3438.minor diff --git a/newsfragments/3438.minor b/newsfragments/3438.minor new file mode 100644 index 000000000..e69de29bb From b218b5426ba7028675c73968dc15cf1ef9bffd2d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 25 Sep 2020 09:31:53 -0400 Subject: [PATCH 5/8] docstrings --- src/allmydata/test/web/test_introducer.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index b752022df..a22f36dc3 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -45,6 +45,22 @@ from ..common_web import ( @defer.inlineCallbacks def create_introducer_webish(reactor, port_assigner, basedir): + """ + Create and start an introducer node and return it and its ``WebishServer`` + service. + + :param reactor: The reactor to use to allow the introducer node to use to + listen for connections. + + :param SameProcessStreamEndpointAssigner port_assigner: The assigner to + use to assign a listening port for the introducer node. + + :param bytes basedir: A non-existant path where the introducer node will + be created. + + :return Deferred[(_IntroducerNode, WebishServer)]: A Deferred that fires + with the node and its webish service. + """ node.create_node_dir(basedir, "testing") _, port_endpoint = port_assigner.assign(reactor) with open(join(basedir, "tahoe.cfg"), "w") as f: @@ -64,6 +80,9 @@ def create_introducer_webish(reactor, port_assigner, basedir): class IntroducerWeb(unittest.TestCase): + """ + Tests for web-facing functionality of an introducer node. + """ def setUp(self): self.node = None self.port_assigner = SameProcessStreamEndpointAssigner() From 05def35c266be6aa0f5b98606c7da432f2753892 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 25 Sep 2020 09:31:57 -0400 Subject: [PATCH 6/8] nicer cleanup plus a comment --- src/allmydata/test/web/test_introducer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index a22f36dc3..a92764894 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -88,9 +88,9 @@ class IntroducerWeb(unittest.TestCase): self.port_assigner = SameProcessStreamEndpointAssigner() self.port_assigner.setUp() self.addCleanup(self.port_assigner.tearDown) - - def tearDown(self): - return flushEventualQueue(None) + # Anything using Foolscap leaves some timer trash in the reactor that + # we have to arrange to have cleaned up. + self.addCleanup(lambda: flushEventualQueue(None)) @defer.inlineCallbacks def test_welcome(self): From 44cc42351fbf54cedeeb3f8c8201f2040d4c02c5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 25 Sep 2020 09:32:06 -0400 Subject: [PATCH 7/8] remove duplicate request --- src/allmydata/test/web/test_introducer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index a92764894..af1ecf8c2 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -125,8 +125,6 @@ class IntroducerWeb(unittest.TestCase): url = "http://localhost:%d/" % (ws.getPortnum(),) res = yield do_http("get", url) soup = BeautifulSoup(res, 'html5lib') - res = yield do_http("get", url) - soup = BeautifulSoup(res, 'html5lib') assert_soup_has_text( self, soup, From 8cdf5cbf26d573a297ece80dfab817fd81396212 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 25 Sep 2020 11:09:55 -0400 Subject: [PATCH 8/8] unused import --- src/allmydata/test/web/test_introducer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index af1ecf8c2..5fdff47ad 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,6 +1,5 @@ import json from os.path import join -from os import makedirs from bs4 import BeautifulSoup