From 7396130c0a6cc3b9fa05765bf885a8d103915360 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:01 -0400 Subject: [PATCH 1/7] Integration test for I2P. --- integration/test_i2p.py | 239 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 integration/test_i2p.py diff --git a/integration/test_i2p.py b/integration/test_i2p.py new file mode 100644 index 000000000..e87ba28e2 --- /dev/null +++ b/integration/test_i2p.py @@ -0,0 +1,239 @@ +""" +Integration tests for I2P support. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import sys +from os.path import join, exists +from os import mkdir +from time import sleep + +if PY2: + def which(path): + # This will result in skipping I2P tests on Python 2. Oh well. + return None +else: + from shutil import which + +from eliot import log_call + +import pytest +import pytest_twisted + +from . import util + +from twisted.python.filepath import ( + FilePath, +) +from twisted.internet.error import ProcessExitedAlready + +from allmydata.test.common import ( + write_introducer, +) + +if which("docker") is None: + pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) + + +@pytest.fixture(scope="session") +def i2p_network(reactor, temp_dir, request): + """Fixture to start up local i2pd.""" + proto = util._MagicTextProtocol("ephemeral keys") + reactor.spawnProcess( + proto, + which("docker"), + ( + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + # Bad URL for reseeds, so it can't talk to other routers. + "--reseed.urls", "http://localhost:1/", + ), + ) + pytest_twisted.blockon(proto.magic_seen) + + def cleanup(): + try: + proto.transport.signalProcess("INT") + util.block_with_timeout(proto.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + +@pytest.fixture(scope='session') +@log_call( + action_type=u"integration:i2p:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def i2p_introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer_i2p +web.port = 4561 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer_i2p') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = util._ProcessExitedProtocol() + util._tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=i2p', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old + # "start" command. + protocol = util._MagicTextProtocol('introducer running') + transport = util._tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def cleanup(): + try: + transport.signalProcess('TERM') + util.block_with_timeout(protocol.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + pytest_twisted.blockon(protocol.magic_seen) + return transport + + +@pytest.fixture(scope='session') +def i2p_introducer_furl(i2p_introducer, temp_dir): + furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl + + +@pytest_twisted.inlineCallbacks +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): + yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + # ensure both nodes are connected to "a grid" by uploading + # something via carol, and retrieve it using dave. + gold_path = join(temp_dir, "gold") + with open(gold_path, "w") as f: + f.write( + "The object-capability model is a computer security model. A " + "capability describes a transferable right to perform one (or " + "more) operations on a given object." + ) + # XXX could use treq or similar to POST these to their respective + # WUIs instead ... + + proto = util._CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'carol'), + 'put', gold_path, + ) + ) + yield proto.done + cap = proto.output.getvalue().strip().split()[-1] + print("TEH CAP!", cap) + + proto = util._CollectOutputProtocol(capture_stderr=False) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'dave'), + 'get', cap, + ) + ) + yield proto.done + + dave_got = proto.output.getvalue().strip() + assert dave_got == open(gold_path, 'rb').read().strip() + + +@pytest_twisted.inlineCallbacks +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): + node_dir = FilePath(temp_dir).child(name) + web_port = "tcp:{}:interface=localhost".format(control_port + 2000) + + print("creating", node_dir.path) + node_dir.makedirs() + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--listen', 'i2p', + node_dir.path, + ) + ) + yield proto.done + + + # Which services should this client connect to? + write_introducer(node_dir, "default", introducer_furl) + with node_dir.child('tahoe.cfg').open('w') as f: + node_config = ''' +[node] +nickname = %(name)s +web.port = %(web_port)s +web.static = public_html +log_gatherer.furl = %(log_furl)s + +[i2p] +enabled = true + +[client] +shares.needed = 1 +shares.happy = 1 +shares.total = 2 + +''' % { + 'name': name, + 'web_port': web_port, + 'log_furl': flog_gatherer, +} + node_config = node_config.encode("utf-8") + f.write(node_config) + + print("running") + yield util._run_node(reactor, node_dir.path, request, None) + print("okay, launched") From dbd2e7f9730f47184213bdd926e639e99dea0f0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:26 -0400 Subject: [PATCH 2/7] News file. --- newsfragments/3743.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3743.minor diff --git a/newsfragments/3743.minor b/newsfragments/3743.minor new file mode 100644 index 000000000..e69de29bb From ddca3e9ab8139518e611ff3f7dd7e84e399c1a0e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:21:40 -0400 Subject: [PATCH 3/7] At this point all integration tests are expected to pass on Python 3. --- src/allmydata/util/_python3.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8025582f2..f65e0aaf9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,21 +17,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -PORTED_INTEGRATION_TESTS = [ - "integration.test_aaa_aardvark", - "integration.test_servers_of_happiness", - "integration.test_sftp", - "integration.test_streaming_logs", - "integration.test_tor", - "integration.test_web", -] - -PORTED_INTEGRATION_MODULES = [ - "integration", - "integration.conftest", - "integration.util", -] - # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata", From 982ac3cc33aad62eb647738245a3b9e52882c26f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 29 Jul 2021 10:02:02 -0400 Subject: [PATCH 4/7] Timeout if i2pd never starts. --- integration/test_i2p.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index e87ba28e2..3f2133f6f 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -56,7 +56,6 @@ def i2p_network(reactor, temp_dir, request): "--reseed.urls", "http://localhost:1/", ), ) - pytest_twisted.blockon(proto.magic_seen) def cleanup(): try: @@ -66,6 +65,8 @@ def i2p_network(reactor, temp_dir, request): pass request.addfinalizer(cleanup) + util.block_with_timeout(proto.magic_seen, reactor, timeout=30) + @pytest.fixture(scope='session') @log_call( From 97522641d669662971ce29fb7678a4b6eec50d8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:06:28 -0400 Subject: [PATCH 5/7] Skip on Windows. --- integration/test_i2p.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 3f2133f6f..4f212f5fb 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -41,6 +41,10 @@ from allmydata.test.common import ( if which("docker") is None: pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) +# Docker on Windows machines sometimes expects Windows-y Docker images, so just +# don't bother. +if sys.platform.startswith('win'): + pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) @pytest.fixture(scope="session") From ce2363e3ded009bc3604a88af6445903dd45ea81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:08:56 -0400 Subject: [PATCH 6/7] More aggressively shut down i2pd and other i2p-related processes. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 4f212f5fb..1aa7a5d99 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -47,7 +47,7 @@ if sys.platform.startswith('win'): pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) -@pytest.fixture(scope="session") +@pytest.fixture def i2p_network(reactor, temp_dir, request): """Fixture to start up local i2pd.""" proto = util._MagicTextProtocol("ephemeral keys") @@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request): def cleanup(): try: - proto.transport.signalProcess("INT") + proto.transport.signalProcess("KILL") util.block_with_timeout(proto.exited, reactor) except ProcessExitedAlready: pass @@ -72,7 +72,7 @@ def i2p_network(reactor, temp_dir, request): util.block_with_timeout(proto.magic_seen, reactor, timeout=30) -@pytest.fixture(scope='session') +@pytest.fixture @log_call( action_type=u"integration:i2p:introducer", include_args=["temp_dir", "flog_gatherer"], @@ -133,7 +133,7 @@ log_gatherer.furl = {log_furl} return transport -@pytest.fixture(scope='session') +@pytest.fixture def i2p_introducer_furl(i2p_introducer, temp_dir): furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') while not exists(furl_fname): From 83cc42a7c3ed78ac2f6ac9946273d11cfdc0e5fc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:27:41 -0400 Subject: [PATCH 7/7] Choose node names that won't conflict. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 1aa7a5d99..f0b06f1e2 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -145,8 +145,8 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): - yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) - yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -165,7 +165,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'carol'), + '-d', join(temp_dir, 'carol_i2p'), 'put', gold_path, ) ) @@ -179,7 +179,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'dave'), + '-d', join(temp_dir, 'dave_i2p'), 'get', cap, ) )