diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 62fa78283..503bd3c34 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,8 +1,10 @@ import os, re, sys, time, json +from functools import partial from twisted.trial import unittest from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.application import service import allmydata @@ -381,6 +383,36 @@ def flush_but_dont_ignore(res): d.addCallback(_done) return d +def _render_config(config): + """ + Convert a ``dict`` of ``dict`` of ``bytes`` to an ini-format string. + """ + return "\n\n".join(list( + _render_config_section(k, v) + for (k, v) + in config.items() + )) + +def _render_config_section(heading, values): + """ + Convert a ``bytes`` heading and a ``dict`` of ``bytes`` to an ini-format + section as ``bytes``. + """ + return "[{}]\n{}\n".format( + heading, _render_section_values(values) + ) + +def _render_section_values(values): + """ + Convert a ``dict`` of ``bytes`` to the body of an ini-format section as + ``bytes``. + """ + return "\n".join(list( + "{} = {}".format(k, v) + for (k, v) + in sorted(values.items()) + )) + class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # SystemTestMixin tests tend to be a lot of work, and we have a few @@ -409,37 +441,63 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): s.setServiceParent(self.sparent) return s - def set_up_nodes(self, NUMCLIENTS=5, use_stats_gatherer=False): - self.numclients = NUMCLIENTS + def _create_introducer(self): iv_dir = self.getdir("introducer") if not os.path.isdir(iv_dir): + introducer_config = ( + u"[node]\n" + u"nickname = introducer \N{BLACK SMILING FACE}\n" + u"web.port = tcp:0:interface=127.0.0.1\n" + ).encode("utf-8") + fileutil.make_dirs(iv_dir) - fileutil.write(os.path.join(iv_dir, 'tahoe.cfg'), - "[node]\n" + - u"nickname = introducer \u263A\n".encode('utf-8') + - "web.port = tcp:0:interface=127.0.0.1\n") + fileutil.write( + os.path.join(iv_dir, 'tahoe.cfg'), + introducer_config, + ) if SYSTEM_TEST_CERTS: os.mkdir(os.path.join(iv_dir, "private")) f = open(os.path.join(iv_dir, "private", "node.pem"), "w") f.write(SYSTEM_TEST_CERTS[0]) f.close() - iv = create_introducer(basedir=iv_dir) - self.introducer = self.add_service(iv) - self._get_introducer_web() - d = defer.succeed(None) - if use_stats_gatherer: - d.addCallback(self._set_up_stats_gatherer) - d.addCallback(self._set_up_nodes_2) - if use_stats_gatherer: - d.addCallback(self._grab_stats) - return d + return create_introducer(basedir=iv_dir) def _get_introducer_web(self): - f = open(os.path.join(self.getdir("introducer"), "node.url"), "r") - self.introweb_url = f.read().strip() - f.close() + with open(os.path.join(self.getdir("introducer"), "node.url"), "r") as f: + return f.read().strip() - def _set_up_stats_gatherer(self, res): + @inlineCallbacks + def set_up_nodes(self, NUMCLIENTS=5, use_stats_gatherer=False): + """ + Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All + of the nodes are running in this process. + + As a side-effect, set: + + * ``numclients`` to ``NUMCLIENTS`` + * ``introducer`` to the ``_IntroducerNode`` instance + * ``introweb_url`` to the introducer's HTTP API endpoint. + + :param int NUMCLIENTS: The number of client nodes to create. + + :param bool use_stats_gatherer: If ``True`` then also create a stats + gatherer and configure the other nodes to use it. + + :return: A ``Deferred`` that fires when the nodes have connected to + each other. + """ + self.numclients = NUMCLIENTS + + self.introducer = self.add_service(self._create_introducer()) + self.introweb_url = self._get_introducer_web() + + if use_stats_gatherer: + yield self._set_up_stats_gatherer() + yield self._set_up_client_nodes() + if use_stats_gatherer: + yield self._grab_stats() + + def _set_up_stats_gatherer(self): statsdir = self.getdir("stats_gatherer") fileutil.make_dirs(statsdir) portnum = iputil.allocate_tcp_port() @@ -461,58 +519,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): d.addCallback(get_furl) return d - def _set_up_nodes_2(self, res): + @inlineCallbacks + def _set_up_client_nodes(self): q = self.introducer self.introducer_furl = q.introducer_url self.clients = [] basedirs = [] for i in range(self.numclients): - basedir = self.getdir("client%d" % i) - basedirs.append(basedir) - fileutil.make_dirs(os.path.join(basedir, "private")) - if len(SYSTEM_TEST_CERTS) > (i+1): - f = open(os.path.join(basedir, "private", "node.pem"), "w") - f.write(SYSTEM_TEST_CERTS[i+1]) - f.close() - - config = "[client]\n" - if i != 1: - # clients[1] uses private/introducers.yaml, not tahoe.cfg - config += "introducer.furl = %s\n" % self.introducer_furl - if self.stats_gatherer_furl: - config += "stats_gatherer.furl = %s\n" % self.stats_gatherer_furl - - nodeconfig = "[node]\n" - nodeconfig += (u"nickname = client %d \u263A\n" % (i,)).encode('utf-8') - tub_port = iputil.allocate_tcp_port() - # Don't let it use AUTO: there's no need for tests to use - # anything other than 127.0.0.1 - nodeconfig += "tub.port = tcp:%d\n" % tub_port - nodeconfig += "tub.location = tcp:127.0.0.1:%d\n" % tub_port - - if i == 0: - # clients[0] runs a webserver and a helper - config += nodeconfig - config += "web.port = tcp:0:interface=127.0.0.1\n" - config += "timeout.keepalive = 600\n" - config += "[helper]\n" - config += "enabled = True\n" - elif i == 1: - # clients[1] uses private/introducers.yaml, not tahoe.cfg - iyaml = ("introducers:\n" - " petname2:\n" - " furl: %s\n") % self.introducer_furl - iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") - fileutil.write(iyaml_fn, iyaml) - elif i == 3: - # clients[3] runs a webserver and uses a helper - config += nodeconfig - config += "web.port = tcp:0:interface=127.0.0.1\n" - config += "timeout.disconnect = 1800\n" - else: - config += nodeconfig - - fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) + basedirs.append((yield self._set_up_client_node(i))) # give subclasses a chance to append lines to the node's tahoe.cfg # files before they are launched. @@ -541,22 +555,87 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): self.clients.append(c) c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) log.msg("STARTING") - d = self.wait_for_connections() - def _connected(res): - log.msg("CONNECTED") - # now find out where the web port was - self.webish_url = self.clients[0].getServiceNamed("webish").getURL() - if self.numclients >=4: - # and the helper-using webport - self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() - d.addCallback(_connected) - return d + yield self.wait_for_connections() + log.msg("CONNECTED") + # now find out where the web port was + self.webish_url = self.clients[0].getServiceNamed("webish").getURL() + if self.numclients >=4: + # and the helper-using webport + self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() + + def _generate_config(self, which, basedir): + config = {} + + except1 = set(range(self.numclients)) - {1} + feature_matrix = { + # client 1 uses private/introducers.yaml, not tahoe.cfg + ("client", "introducer.furl"): except1, + ("client", "nickname"): except1, + + ("node", "tub.port"): except1, + ("node", "tub.location"): except1, + # clients 0 runs a webserver and a helper + ("node", "web.port"): {0, 3}, + ("node", "timeout.keepalive"): {0}, + ("node", "timeout.disconnect"): {3}, + + ("helper", "enabled"): {0}, + } + + def setconf(config, which, section, feature, value): + if which in feature_matrix.get((section, feature), {which}): + if isinstance(value, unicode): + value = value.encode("utf-8") + config.setdefault(section, {})[feature] = value + + setclient = partial(setconf, config, which, "client") + setnode = partial(setconf, config, which, "node") + sethelper = partial(setconf, config, which, "helper") + + setclient("introducer.furl", self.introducer_furl) + setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) + + if self.stats_gatherer_furl: + setclient("stats_gatherer.furl", self.stats_gatherer_furl) + + tub_port = iputil.allocate_tcp_port() + # Don't let it use AUTO: there's no need for tests to use + # anything other than 127.0.0.1 + setnode("tub.port", "tcp:%d" % (tub_port,)) + setnode("tub.location", "tcp:127.0.0.1:%d" % (tub_port,)) + + setnode("web.port", "tcp:0:interface=127.0.0.1") + setnode("timeout.keepalive", "600") + setnode("timeout.disconnect", "1800") + + sethelper("enabled", "True") + + if which == 1: + # clients[1] uses private/introducers.yaml, not tahoe.cfg + iyaml = ("introducers:\n" + " petname2:\n" + " furl: %s\n") % self.introducer_furl + iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") + fileutil.write(iyaml_fn, iyaml) + + return _render_config(config) + + def _set_up_client_node(self, which): + basedir = self.getdir("client%d" % (which,)) + fileutil.make_dirs(os.path.join(basedir, "private")) + if len(SYSTEM_TEST_CERTS) > (which + 1): + f = open(os.path.join(basedir, "private", "node.pem"), "w") + f.write(SYSTEM_TEST_CERTS[which + 1]) + f.close() + config = self._generate_config(which, basedir) + fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) + return basedir def _set_up_nodes_extra_config(self): # for overriding by subclasses pass - def _grab_stats(self, res): + def _grab_stats(self): d = self.stats_gatherer.poll() return d