From b96122a8ffb819c55f47ea946cab03a4e22e60e5 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 5 Oct 2016 23:03:35 -0600 Subject: [PATCH] Adding to the py.test integration tests, this: - (on travis) installs Tor - installs Chutney - uses it to build a local Tor test-network - set up an introducer on this test-network - sets up two storage servers on this test-network - proves that one can add a file, and the other can download it I also mark the two tests that occasionally fail as expected failures for now --- .travis.yml | 1 + integration/__init__.py | 1 + integration/conftest.py | 303 ++++++++++++++++++------------------- integration/install-tor.sh | 49 ++++++ integration/test_smoke.py | 11 +- integration/test_tor.py | 123 +++++++++++++++ integration/util.py | 188 ++++++++++++++++++++++- 7 files changed, 518 insertions(+), 158 deletions(-) create mode 100644 integration/__init__.py create mode 100755 integration/install-tor.sh create mode 100644 integration/test_tor.py diff --git a/.travis.yml b/.travis.yml index b12a34ce1..127fd5854 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ install: script: - tox -e codechecks - if [ "${TRAVIS_OS_NAME}" = "osx" ]; then tox; else tox -e coverage; fi + - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then /bin/bash integration/install-tor.sh; fi - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then tox -e integration; fi after_success: - if [ "${TRAVIS_OS_NAME}" = "linux" ]; then codecov; fi diff --git a/integration/__init__.py b/integration/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/integration/__init__.py @@ -0,0 +1 @@ + diff --git a/integration/conftest.py b/integration/conftest.py index 9992363bc..bc0cbc67f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -6,16 +6,22 @@ from sys import stdout as _stdout from os import mkdir, listdir, unlink from os.path import join, abspath, curdir, exists from tempfile import mkdtemp, mktemp -from StringIO import StringIO from shutilwhich import which from twisted.internet.defer import Deferred, DeferredList from twisted.internet.task import deferLater -from twisted.internet.protocol import ProcessProtocol -from twisted.internet.error import ProcessExitedAlready, ProcessDone +from twisted.internet.error import ProcessExitedAlready import pytest +from util import _CollectOutputProtocol +from util import _MagicTextProtocol +from util import _DumpOutputProtocol +from util import _ProcessExitedProtocol +from util import _create_node +from util import _run_node + + pytest_plugins = 'pytest_twisted' # pytest customization hooks @@ -70,94 +76,6 @@ def flog_binary(): return which('flogtool') -class _ProcessExitedProtocol(ProcessProtocol): - """ - Internal helper that .callback()s on self.done when the process - exits (for any reason). - """ - - def __init__(self): - self.done = Deferred() - - def processEnded(self, reason): - self.done.callback(None) - - -class _CollectOutputProtocol(ProcessProtocol): - """ - Internal helper. Collects all output (stdout + stderr) into - self.output, and callback's on done with all of it after the - process exits (for any reason). - """ - def __init__(self): - self.done = Deferred() - self.output = StringIO() - - def processEnded(self, reason): - if not self.done.called: - self.done.callback(self.output.getvalue()) - - def processExited(self, reason): - if not isinstance(reason.value, ProcessDone): - self.done.errback(reason) - - def outReceived(self, data): - self.output.write(data) - - def errReceived(self, data): - print("ERR", data) - self.output.write(data) - - -class _DumpOutputProtocol(ProcessProtocol): - """ - Internal helper. - """ - def __init__(self, f): - self.done = Deferred() - self._out = f if f is not None else sys.stdout - - def processEnded(self, reason): - if not self.done.called: - self.done.callback(None) - - def processExited(self, reason): - if not isinstance(reason.value, ProcessDone): - self.done.errback(reason) - - def outReceived(self, data): - self._out.write(data) - - def errReceived(self, data): - self._out.write(data) - - -class _MagicTextProtocol(ProcessProtocol): - """ - Internal helper. Monitors all stdout looking for a magic string, - and then .callback()s on self.done and .errback's if the process exits - """ - - def __init__(self, magic_text): - self.magic_seen = Deferred() - self.exited = Deferred() - self._magic_text = magic_text - self._output = StringIO() - - def processEnded(self, reason): - self.exited.callback(None) - - def outReceived(self, data): - sys.stdout.write(data) - self._output.write(data) - if not self.magic_seen.called and self._magic_text in self._output.getvalue(): - print("Saw '{}' in the logs".format(self._magic_text)) - self.magic_seen.callback(None) - - def errReceived(self, data): - sys.stdout.write(data) - - @pytest.fixture(scope='session') def flog_gatherer(reactor, temp_dir, flog_binary, request): out_protocol = _CollectOutputProtocol() @@ -283,24 +201,51 @@ def introducer_furl(introducer, temp_dir): return furl -def _run_node(reactor, node_dir, request, magic_text): - if magic_text is None: - magic_text = "client running" - protocol = _MagicTextProtocol(magic_text) +@pytest.fixture(scope='session') +def tor_introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer_tor +web.port = 4561 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer_tor') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = _ProcessExitedProtocol() + reactor.spawnProcess( + done_proto, + sys.executable, + ( + sys.executable, '-m', 'allmydata.scripts.runner', + 'create-introducer', + '--tor-control-port', 'tcp:localhost:8010', + '--listen=tor', + intro_dir, + ), + ) + pytest.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) # on windows, "tahoe start" means: run forever in the foreground, # but on linux it means daemonize. "tahoe run" is consistent # between platforms. + protocol = _MagicTextProtocol('introducer running') process = reactor.spawnProcess( protocol, sys.executable, ( sys.executable, '-m', 'allmydata.scripts.runner', 'run', - node_dir, + intro_dir, ), ) - process.exited = protocol.exited def cleanup(): try: @@ -310,66 +255,18 @@ def _run_node(reactor, node_dir, request, magic_text): pass request.addfinalizer(cleanup) - # we return the 'process' ITransport instance - # XXX abusing the Deferred; should use .when_magic_seen() or something? - protocol.magic_seen.addCallback(lambda _: process) - return protocol.magic_seen + pytest.blockon(protocol.magic_seen) + return process -def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port, storage=True, magic_text=None): - """ - Helper to create a single node, run it and return the instance - spawnProcess returned (ITransport) - """ - node_dir = join(temp_dir, name) - if web_port is None: - web_port = '' - if not exists(node_dir): - print("creating", node_dir) - mkdir(node_dir) - done_proto = _ProcessExitedProtocol() - args = [ - sys.executable, '-m', 'allmydata.scripts.runner', - 'create-node', - '--nickname', name, - '--introducer', introducer_furl, - '--hostname', 'localhost', - '--listen', 'tcp', - ] - if not storage: - args.append('--no-storage') - args.append(node_dir) - - reactor.spawnProcess( - done_proto, - sys.executable, - args, - ) - pytest.blockon(done_proto.done) - - with open(join(node_dir, 'tahoe.cfg'), 'w') as f: - f.write(''' -[node] -nickname = %(name)s -web.port = %(web_port)s -web.static = public_html -log_gatherer.furl = %(log_furl)s - -[client] -# Which services should this client connect to? -introducer.furl = %(furl)s -shares.needed = 2 -shares.happy = 3 -shares.total = 4 - -''' % { - 'name': name, - 'furl': introducer_furl, - 'web_port': web_port, - 'log_furl': flog_gatherer, -}) - - return _run_node(reactor, node_dir, request, magic_text) +@pytest.fixture(scope='session') +def tor_introducer_furl(tor_introducer, temp_dir): + furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + time.sleep(.1) + furl = open(furl_fname, 'r').read() + return furl @pytest.fixture(scope='session') @@ -502,3 +399,99 @@ def magic_folder(reactor, alice_invite, alice, bob, temp_dir, request): magic_text = 'Completed initial Magic Folder scan successfully' pytest.blockon(_run_node(reactor, bob_dir, request, magic_text)) return (join(temp_dir, 'magic-alice'), join(temp_dir, 'magic-bob')) + + +@pytest.fixture(scope='session') +def chutney(reactor, temp_dir): + chutney_dir = join(temp_dir, 'chutney') + mkdir(chutney_dir) + + # TODO: + + # check for 'tor' binary explicitly and emit a "skip" if we can't + # find it + + # XXX yuck! should add a setup.py to chutney so we can at least + # "pip install " and/or depend on chutney in "pip + # install -e .[dev]" (i.e. in the 'dev' extra) + proto = _DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + '/usr/bin/git', + ( + '/usr/bin/git', 'clone', '--depth=1', + 'https://git.torproject.org/chutney.git', + chutney_dir, + ) + ) + pytest.blockon(proto.done) + return chutney_dir + + +@pytest.fixture(scope='session') +def tor_network(reactor, temp_dir, chutney, request): + # this is the actual "chutney" script at the root of a chutney checkout + chutney_dir = chutney + chut = join(chutney_dir, 'chutney') + + # now, as per Chutney's README, we have to create the network + # ./chutney configure networks/basic + # ./chutney start networks/basic + + proto = _DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'chutney.TorNet', 'configure', + join(chutney_dir, 'networks', 'basic'), + ), + path=join(chutney_dir), + env={"PYTHONPATH": join(chutney_dir, "lib")}, + ) + pytest.blockon(proto.done) + + proto = _DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'chutney.TorNet', 'start', + join(chutney_dir, 'networks', 'basic'), + ), + path=join(chutney_dir), + env={"PYTHONPATH": join(chutney_dir, "lib")}, + ) + pytest.blockon(proto.done) + + # print some useful stuff + proto = _CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'chutney.TorNet', 'status', + join(chutney_dir, 'networks', 'basic'), + ), + path=join(chutney_dir), + env={"PYTHONPATH": join(chutney_dir, "lib")}, + ) + pytest.blockon(proto.done) + + def cleanup(): + print("Tearing down Chutney Tor network") + proto = _CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-m', 'chutney.TorNet', 'stop', + join(chutney_dir, 'networks', 'basic'), + ), + path=join(chutney_dir), + env={"PYTHONPATH": join(chutney_dir, "lib")}, + ) + pytest.blockon(proto.done) + request.addfinalizer(cleanup) + + return chut diff --git a/integration/install-tor.sh b/integration/install-tor.sh new file mode 100755 index 000000000..02124b1d9 --- /dev/null +++ b/integration/install-tor.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Script to install Tor + +set -ex +echo "deb http://deb.torproject.org/torproject.org precise main" | sudo tee -a /etc/apt/sources.list +echo "deb-src http://deb.torproject.org/torproject.org precise main" | sudo tee -a /etc/apt/sources.list + +# Install Tor repo signing key +sudo apt-key add - <