mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-03-23 12:15:16 +00:00
system test setup broken into manageable pieces
`set_up_nodes` and `_set_up_nodes_2` now a little easier to understand
This commit is contained in:
Jean-Paul Calderone
parent
cc3897a49d
commit
f576575f11
@ -1,8 +1,10 @@
|
||||
|
||||
import os, re, sys, time, json
|
||||
from functools import partial
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
from twisted.application import service
|
||||
|
||||
import allmydata
|
||||
@ -381,6 +383,36 @@ def flush_but_dont_ignore(res):
|
||||
d.addCallback(_done)
|
||||
return d
|
||||
|
||||
def _render_config(config):
|
||||
"""
|
||||
Convert a ``dict`` of ``dict`` of ``bytes`` to an ini-format string.
|
||||
"""
|
||||
return "\n\n".join(list(
|
||||
_render_config_section(k, v)
|
||||
for (k, v)
|
||||
in config.items()
|
||||
))
|
||||
|
||||
def _render_config_section(heading, values):
|
||||
"""
|
||||
Convert a ``bytes`` heading and a ``dict`` of ``bytes`` to an ini-format
|
||||
section as ``bytes``.
|
||||
"""
|
||||
return "[{}]\n{}\n".format(
|
||||
heading, _render_section_values(values)
|
||||
)
|
||||
|
||||
def _render_section_values(values):
|
||||
"""
|
||||
Convert a ``dict`` of ``bytes`` to the body of an ini-format section as
|
||||
``bytes``.
|
||||
"""
|
||||
return "\n".join(list(
|
||||
"{} = {}".format(k, v)
|
||||
for (k, v)
|
||||
in sorted(values.items())
|
||||
))
|
||||
|
||||
class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
|
||||
# SystemTestMixin tests tend to be a lot of work, and we have a few
|
||||
@ -409,37 +441,63 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
s.setServiceParent(self.sparent)
|
||||
return s
|
||||
|
||||
def set_up_nodes(self, NUMCLIENTS=5, use_stats_gatherer=False):
|
||||
self.numclients = NUMCLIENTS
|
||||
def _create_introducer(self):
|
||||
iv_dir = self.getdir("introducer")
|
||||
if not os.path.isdir(iv_dir):
|
||||
introducer_config = (
|
||||
u"[node]\n"
|
||||
u"nickname = introducer \N{BLACK SMILING FACE}\n"
|
||||
u"web.port = tcp:0:interface=127.0.0.1\n"
|
||||
).encode("utf-8")
|
||||
|
||||
fileutil.make_dirs(iv_dir)
|
||||
fileutil.write(os.path.join(iv_dir, 'tahoe.cfg'),
|
||||
"[node]\n" +
|
||||
u"nickname = introducer \u263A\n".encode('utf-8') +
|
||||
"web.port = tcp:0:interface=127.0.0.1\n")
|
||||
fileutil.write(
|
||||
os.path.join(iv_dir, 'tahoe.cfg'),
|
||||
introducer_config,
|
||||
)
|
||||
if SYSTEM_TEST_CERTS:
|
||||
os.mkdir(os.path.join(iv_dir, "private"))
|
||||
f = open(os.path.join(iv_dir, "private", "node.pem"), "w")
|
||||
f.write(SYSTEM_TEST_CERTS[0])
|
||||
f.close()
|
||||
iv = create_introducer(basedir=iv_dir)
|
||||
self.introducer = self.add_service(iv)
|
||||
self._get_introducer_web()
|
||||
d = defer.succeed(None)
|
||||
if use_stats_gatherer:
|
||||
d.addCallback(self._set_up_stats_gatherer)
|
||||
d.addCallback(self._set_up_nodes_2)
|
||||
if use_stats_gatherer:
|
||||
d.addCallback(self._grab_stats)
|
||||
return d
|
||||
return create_introducer(basedir=iv_dir)
|
||||
|
||||
def _get_introducer_web(self):
|
||||
f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
|
||||
self.introweb_url = f.read().strip()
|
||||
f.close()
|
||||
with open(os.path.join(self.getdir("introducer"), "node.url"), "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
def _set_up_stats_gatherer(self, res):
|
||||
@inlineCallbacks
|
||||
def set_up_nodes(self, NUMCLIENTS=5, use_stats_gatherer=False):
|
||||
"""
|
||||
Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All
|
||||
of the nodes are running in this process.
|
||||
|
||||
As a side-effect, set:
|
||||
|
||||
* ``numclients`` to ``NUMCLIENTS``
|
||||
* ``introducer`` to the ``_IntroducerNode`` instance
|
||||
* ``introweb_url`` to the introducer's HTTP API endpoint.
|
||||
|
||||
:param int NUMCLIENTS: The number of client nodes to create.
|
||||
|
||||
:param bool use_stats_gatherer: If ``True`` then also create a stats
|
||||
gatherer and configure the other nodes to use it.
|
||||
|
||||
:return: A ``Deferred`` that fires when the nodes have connected to
|
||||
each other.
|
||||
"""
|
||||
self.numclients = NUMCLIENTS
|
||||
|
||||
self.introducer = self.add_service(self._create_introducer())
|
||||
self.introweb_url = self._get_introducer_web()
|
||||
|
||||
if use_stats_gatherer:
|
||||
yield self._set_up_stats_gatherer()
|
||||
yield self._set_up_client_nodes()
|
||||
if use_stats_gatherer:
|
||||
yield self._grab_stats()
|
||||
|
||||
def _set_up_stats_gatherer(self):
|
||||
statsdir = self.getdir("stats_gatherer")
|
||||
fileutil.make_dirs(statsdir)
|
||||
portnum = iputil.allocate_tcp_port()
|
||||
@ -461,58 +519,14 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
d.addCallback(get_furl)
|
||||
return d
|
||||
|
||||
def _set_up_nodes_2(self, res):
|
||||
@inlineCallbacks
|
||||
def _set_up_client_nodes(self):
|
||||
q = self.introducer
|
||||
self.introducer_furl = q.introducer_url
|
||||
self.clients = []
|
||||
basedirs = []
|
||||
for i in range(self.numclients):
|
||||
basedir = self.getdir("client%d" % i)
|
||||
basedirs.append(basedir)
|
||||
fileutil.make_dirs(os.path.join(basedir, "private"))
|
||||
if len(SYSTEM_TEST_CERTS) > (i+1):
|
||||
f = open(os.path.join(basedir, "private", "node.pem"), "w")
|
||||
f.write(SYSTEM_TEST_CERTS[i+1])
|
||||
f.close()
|
||||
|
||||
config = "[client]\n"
|
||||
if i != 1:
|
||||
# clients[1] uses private/introducers.yaml, not tahoe.cfg
|
||||
config += "introducer.furl = %s\n" % self.introducer_furl
|
||||
if self.stats_gatherer_furl:
|
||||
config += "stats_gatherer.furl = %s\n" % self.stats_gatherer_furl
|
||||
|
||||
nodeconfig = "[node]\n"
|
||||
nodeconfig += (u"nickname = client %d \u263A\n" % (i,)).encode('utf-8')
|
||||
tub_port = iputil.allocate_tcp_port()
|
||||
# Don't let it use AUTO: there's no need for tests to use
|
||||
# anything other than 127.0.0.1
|
||||
nodeconfig += "tub.port = tcp:%d\n" % tub_port
|
||||
nodeconfig += "tub.location = tcp:127.0.0.1:%d\n" % tub_port
|
||||
|
||||
if i == 0:
|
||||
# clients[0] runs a webserver and a helper
|
||||
config += nodeconfig
|
||||
config += "web.port = tcp:0:interface=127.0.0.1\n"
|
||||
config += "timeout.keepalive = 600\n"
|
||||
config += "[helper]\n"
|
||||
config += "enabled = True\n"
|
||||
elif i == 1:
|
||||
# clients[1] uses private/introducers.yaml, not tahoe.cfg
|
||||
iyaml = ("introducers:\n"
|
||||
" petname2:\n"
|
||||
" furl: %s\n") % self.introducer_furl
|
||||
iyaml_fn = os.path.join(basedir, "private", "introducers.yaml")
|
||||
fileutil.write(iyaml_fn, iyaml)
|
||||
elif i == 3:
|
||||
# clients[3] runs a webserver and uses a helper
|
||||
config += nodeconfig
|
||||
config += "web.port = tcp:0:interface=127.0.0.1\n"
|
||||
config += "timeout.disconnect = 1800\n"
|
||||
else:
|
||||
config += nodeconfig
|
||||
|
||||
fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
|
||||
basedirs.append((yield self._set_up_client_node(i)))
|
||||
|
||||
# give subclasses a chance to append lines to the node's tahoe.cfg
|
||||
# files before they are launched.
|
||||
@ -541,22 +555,87 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
|
||||
self.clients.append(c)
|
||||
c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
|
||||
log.msg("STARTING")
|
||||
d = self.wait_for_connections()
|
||||
def _connected(res):
|
||||
log.msg("CONNECTED")
|
||||
# now find out where the web port was
|
||||
self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
|
||||
if self.numclients >=4:
|
||||
# and the helper-using webport
|
||||
self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
|
||||
d.addCallback(_connected)
|
||||
return d
|
||||
yield self.wait_for_connections()
|
||||
log.msg("CONNECTED")
|
||||
# now find out where the web port was
|
||||
self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
|
||||
if self.numclients >=4:
|
||||
# and the helper-using webport
|
||||
self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
|
||||
|
||||
def _generate_config(self, which, basedir):
|
||||
config = {}
|
||||
|
||||
except1 = set(range(self.numclients)) - {1}
|
||||
feature_matrix = {
|
||||
# client 1 uses private/introducers.yaml, not tahoe.cfg
|
||||
("client", "introducer.furl"): except1,
|
||||
("client", "nickname"): except1,
|
||||
|
||||
("node", "tub.port"): except1,
|
||||
("node", "tub.location"): except1,
|
||||
# clients 0 runs a webserver and a helper
|
||||
("node", "web.port"): {0, 3},
|
||||
("node", "timeout.keepalive"): {0},
|
||||
("node", "timeout.disconnect"): {3},
|
||||
|
||||
("helper", "enabled"): {0},
|
||||
}
|
||||
|
||||
def setconf(config, which, section, feature, value):
|
||||
if which in feature_matrix.get((section, feature), {which}):
|
||||
if isinstance(value, unicode):
|
||||
value = value.encode("utf-8")
|
||||
config.setdefault(section, {})[feature] = value
|
||||
|
||||
setclient = partial(setconf, config, which, "client")
|
||||
setnode = partial(setconf, config, which, "node")
|
||||
sethelper = partial(setconf, config, which, "helper")
|
||||
|
||||
setclient("introducer.furl", self.introducer_furl)
|
||||
setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,))
|
||||
|
||||
if self.stats_gatherer_furl:
|
||||
setclient("stats_gatherer.furl", self.stats_gatherer_furl)
|
||||
|
||||
tub_port = iputil.allocate_tcp_port()
|
||||
# Don't let it use AUTO: there's no need for tests to use
|
||||
# anything other than 127.0.0.1
|
||||
setnode("tub.port", "tcp:%d" % (tub_port,))
|
||||
setnode("tub.location", "tcp:127.0.0.1:%d" % (tub_port,))
|
||||
|
||||
setnode("web.port", "tcp:0:interface=127.0.0.1")
|
||||
setnode("timeout.keepalive", "600")
|
||||
setnode("timeout.disconnect", "1800")
|
||||
|
||||
sethelper("enabled", "True")
|
||||
|
||||
if which == 1:
|
||||
# clients[1] uses private/introducers.yaml, not tahoe.cfg
|
||||
iyaml = ("introducers:\n"
|
||||
" petname2:\n"
|
||||
" furl: %s\n") % self.introducer_furl
|
||||
iyaml_fn = os.path.join(basedir, "private", "introducers.yaml")
|
||||
fileutil.write(iyaml_fn, iyaml)
|
||||
|
||||
return _render_config(config)
|
||||
|
||||
def _set_up_client_node(self, which):
|
||||
basedir = self.getdir("client%d" % (which,))
|
||||
fileutil.make_dirs(os.path.join(basedir, "private"))
|
||||
if len(SYSTEM_TEST_CERTS) > (which + 1):
|
||||
f = open(os.path.join(basedir, "private", "node.pem"), "w")
|
||||
f.write(SYSTEM_TEST_CERTS[which + 1])
|
||||
f.close()
|
||||
config = self._generate_config(which, basedir)
|
||||
fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
|
||||
return basedir
|
||||
|
||||
def _set_up_nodes_extra_config(self):
|
||||
# for overriding by subclasses
|
||||
pass
|
||||
|
||||
def _grab_stats(self, res):
|
||||
def _grab_stats(self):
|
||||
d = self.stats_gatherer.poll()
|
||||
return d
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user