From f19bf8cf86d58457be4e7b4726626f3de5f29dc3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:04:55 -0400 Subject: [PATCH 01/12] Parameterize the options object to the `run_cli` helper --- src/allmydata/test/common_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index d2d20916d..e63c3eef8 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -69,6 +69,9 @@ def run_cli_native(verb, *args, **kwargs): Most code should prefer ``run_cli_unicode`` which deals with all the necessary encoding considerations. + :param runner.Options options: The options instance to use to parse the + given arguments. + :param native_str verb: The command to run. For example, ``"create-node"``. @@ -88,6 +91,7 @@ def run_cli_native(verb, *args, **kwargs): matching native behavior. If True, stdout/stderr are returned as bytes. """ + options = kwargs.pop("options", runner.Options()) nodeargs = kwargs.pop("nodeargs", []) encoding = kwargs.pop("encoding", None) or getattr(sys.stdout, "encoding") or "utf-8" return_bytes = kwargs.pop("return_bytes", False) @@ -134,7 +138,7 @@ def run_cli_native(verb, *args, **kwargs): d.addCallback( partial( runner.parse_or_exit, - runner.Options(), + options, ), stdout=stdout, stderr=stderr, From dffcdf28543ced3bd214370bccbb5b78354d587d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 11 Apr 2022 15:05:32 -0400 Subject: [PATCH 02/12] Clean up the Py2/Py3 boilerplate --- src/allmydata/test/cli/test_invite.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 20d012995..749898b77 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -1,24 +1,12 @@ """ -Ported to Pythn 3. +Tests for ``tahoe invite``. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import mock import json from os.path import join - -try: - from typing import Optional, Sequence -except ImportError: - pass +from typing import Optional, Sequence from twisted.trial import unittest from twisted.internet import defer From bc6dafa999fa9b7d2af8dbab36ed532b74919e6b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:04 -0400 Subject: [PATCH 03/12] Replace monkey-patching of wormhole with a parameter to run_cli --- src/allmydata/scripts/create_node.py | 5 +- src/allmydata/scripts/runner.py | 6 + src/allmydata/scripts/tahoe_invite.py | 6 +- src/allmydata/test/cli/test_invite.py | 508 +++++++++++++--------- src/allmydata/test/cli/wormholetesting.py | 304 +++++++++++++ 5 files changed, 614 insertions(+), 215 deletions(-) create mode 100644 src/allmydata/test/cli/wormholetesting.py diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 4959ed391..5d9da518b 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -37,9 +37,6 @@ from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json -from wormhole import wormhole - - dummy_tac = """ import sys print("Nodes created by Tahoe-LAFS v1.11.0 or later cannot be run by") @@ -377,7 +374,7 @@ def _get_config_via_wormhole(config): relay_url = config.parent['wormhole-server'] print("Connecting to '{}'".format(relay_url), file=out) - wh = wormhole.create( + wh = config.parent.wormhole.create( appid=config.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 145ee6464..a0d8a752b 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -58,11 +58,17 @@ process_control_commands = [ class Options(usage.Options): + """ + :ivar wormhole: An object exposing the magic-wormhole API (mainly a test + hook). + """ # unit tests can override these to point at StringIO instances stdin = sys.stdin stdout = sys.stdout stderr = sys.stderr + from wormhole import wormhole + subCommands = ( create_node.subCommands + admin.subCommands + process_control_commands diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index 09d4cbd59..c5f08f588 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -18,8 +18,6 @@ except ImportError: from twisted.python import usage from twisted.internet import defer, reactor -from wormhole import wormhole - from allmydata.util.encodingutil import argv_to_abspath from allmydata.util import jsonbytes as json from allmydata.scripts.common import get_default_nodedir, get_introducer_furl @@ -44,13 +42,15 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() +wormhole = None + @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout err = options.stderr relay_url = options.parent['wormhole-server'] print("Connecting to '{}'...".format(relay_url), file=out) - wh = wormhole.create( + wh = options.parent.wormhole.create( appid=options.parent['wormhole-invite-appid'], relay_url=relay_url, reactor=reactor, diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 749898b77..c4bb6fd7e 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,62 +2,21 @@ Tests for ``tahoe invite``. """ -import os -import mock import json +import os from os.path import join from typing import Optional, Sequence -from twisted.trial import unittest from twisted.internet import defer +from twisted.trial import unittest + +from ...client import read_config +from ...scripts import runner +from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from ...client import ( - read_config, -) - -class _FakeWormhole(object): - - def __init__(self, outgoing_messages): - self.messages = [] - for o in outgoing_messages: - assert isinstance(o, bytes) - self._outgoing = outgoing_messages - - def get_code(self): - return defer.succeed(u"6-alarmist-tuba") - - def set_code(self, code): - self._code = code - - def get_welcome(self): - return defer.succeed( - { - u"welcome": {}, - } - ) - - def allocate_code(self): - return None - - def send_message(self, msg): - assert isinstance(msg, bytes) - self.messages.append(msg) - - def get_message(self): - return defer.succeed(self._outgoing.pop(0)) - - def close(self): - return defer.succeed(None) - - -def _create_fake_wormhole(outgoing_messages): - outgoing_messages = [ - m.encode("utf-8") if isinstance(m, str) else m - for m in outgoing_messages - ] - return _FakeWormhole(outgoing_messages) +from .wormholetesting import MemoryWormholeServer, memory_server class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -74,41 +33,52 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - self.assertEqual(0, rc) + self.assertEqual(0, rc) - config = read_config(node_dir, u"") - self.assertIn( - "pb://foo", - set( - furl - for (furl, cache) - in config.get_introducer_configuration().values() - ), - ) + config = read_config(node_dir, u"") + self.assertIn( + "pb://foo", + set( + furl + for (furl, cache) + in config.get_introducer_configuration().values() + ), + ) - with open(join(node_dir, 'tahoe.cfg'), 'r') as f: - config = f.read() - self.assertIn(u"somethinghopefullyunique", config) + with open(join(node_dir, 'tahoe.cfg'), 'r') as f: + config = f.read() + self.assertIn(u"somethinghopefullyunique", config) @defer.inlineCallbacks def test_create_node_illegal_option(self): @@ -116,30 +86,41 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v1": {}}}), - json.dumps({ - u"shares-needed": 1, - u"shares-happy": 1, - u"shares-total": 1, - u"nickname": u"somethinghopefullyunique", - u"introducer": u"pb://foo", - u"something-else": u"not allowed", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v1": {}}}, + { + u"shares-needed": 1, + u"shares-happy": 1, + u"shares-total": 1, + u"nickname": u"somethinghopefullyunique", + u"introducer": u"pb://foo", + u"something-else": u"not allowed", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) - rc, out, err = yield run_cli( - "create-client", - "--join", "1-abysmal-ant", - node_dir, - ) + rc, out, err = yield run_cli( + "create-client", + "--join", code, + node_dir, + options=options, + ) - # should still succeed -- just ignores the not-whitelisted - # "something-else" option - self.assertEqual(0, rc) + # should still succeed -- just ignores the not-whitelisted + # "something-else" option + self.assertEqual(0, rc) class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -156,7 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - def _invite_success(self, extra_args=(), tahoe_config=None): + async def _invite_success(self, extra_args=(), tahoe_config=None): # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred """ Exercise an expected-success case of ``tahoe invite``. @@ -178,53 +159,82 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): with open(join(intro_dir, "tahoe.cfg"), "wb") as fobj_cfg: fobj_cfg.write(tahoe_config) - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - extra_args = tuple(extra_args) - - d = run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", - *(extra_args + ("foo",)) + *tuple(extra_args) + ("foo",), + options=options, ) - def done(result): - rc, out, err = result - self.assertEqual(2, len(fake_wh.messages)) - self.assertEqual( - json.loads(fake_wh.messages[0]), + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a proper client abilities message. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) + + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) - invite = json.loads(fake_wh.messages[1]) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - d.addCallback(done) - return d + }, + ) + + # Second, it should have an invitation with a nickname and + # introducer furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) + return invite + + invite, _ = await defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + return invite + @defer.inlineCallbacks def test_invite_success(self): """ successfully send an invite """ - invite = yield self._invite_success(( + invite = yield defer.Deferred.fromCoroutine(self._invite_success(( "--shares-needed", "1", "--shares-happy", "2", "--shares-total", "3", - )) + ))) self.assertEqual( invite["shares-needed"], "1", ) @@ -241,12 +251,12 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): If ``--shares-{needed,happy,total}`` are not given on the command line then the invitation is generated using the configured values. """ - invite = yield self._invite_success(tahoe_config=b""" + invite = yield defer.Deferred.fromCoroutine(self._invite_success(tahoe_config=b""" [client] shares.needed = 2 shares.happy = 4 shares.total = 6 -""") +""")) self.assertEqual( invite["shares-needed"], "2", ) @@ -265,22 +275,20 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v1": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + options = runner.Options() + options.wormhole = None - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn(u"Can't find introducer FURL", out + err) + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn(u"Can't find introducer FURL", out + err) @defer.inlineCallbacks def test_invite_wrong_client_abilities(self): @@ -294,23 +302,51 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"client-v9000": {}}}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send some surprising client abilities. + other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -323,23 +359,52 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server, helper = memory_server() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( + async def server(): + # Run the server side of the invitation process using the CLI. + rc, out, err = await run_cli( "-d", intro_dir, "invite", "--shares-needed", "1", "--shares-happy", "1", "--shares-total", "1", "foo", + options=options, ) self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) + async def client(): + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + + # Send a no-abilities message through to the server. + other_end.send_message(dumps_bytes({})) + + yield defer.gatherResults(map( + defer.Deferred.fromCoroutine, + [client(), server()], + )) + + @defer.inlineCallbacks def test_invite_wrong_server_abilities(self): """ @@ -352,26 +417,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({u"abilities": {u"server-v9000": {}}}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "foo", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + wormhole_server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = wormhole_server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "foo", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'server-v1' in server abilities", out + err) + wormhole = wormhole_server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {u"abilities": {u"server-v9000": {}}}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "foo", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "foo", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'server-v1' in server abilities", out + err) @defer.inlineCallbacks def test_invite_no_server_abilities(self): @@ -385,26 +462,38 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - with mock.patch('allmydata.scripts.create_node.wormhole') as w: - fake_wh = _create_fake_wormhole([ - json.dumps({}), - json.dumps({ - "shares-needed": "1", - "shares-total": "1", - "shares-happy": "1", - "nickname": "bar", - "introducer": "pb://fooblam", - }), - ]) - w.create = mock.Mock(return_value=fake_wh) + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() - rc, out, err = yield run_cli( - "create-client", - "--join", "1-alarmist-tuba", - "bar", - ) - self.assertNotEqual(rc, 0) - self.assertIn("Expected 'abilities' in server introduction", out + err) + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = yield wormhole.get_code() + messages = [ + {}, + { + "shares-needed": "1", + "shares-total": "1", + "shares-happy": "1", + "nickname": "bar", + "introducer": "pb://fooblam", + }, + ] + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + rc, out, err = yield run_cli( + "create-client", + "--join", code, + "bar", + options=options, + ) + self.assertNotEqual(rc, 0) + self.assertIn("Expected 'abilities' in server introduction", out + err) @defer.inlineCallbacks def test_invite_no_nick(self): @@ -413,13 +502,16 @@ shares.total = 6 """ intro_dir = os.path.join(self.basedir, "introducer") - with mock.patch('allmydata.scripts.tahoe_invite.wormhole'): - rc, out, err = yield run_cli( - "-d", intro_dir, - "invite", - "--shares-needed", "1", - "--shares-happy", "1", - "--shares-total", "1", - ) - self.assertTrue(rc) - self.assertIn(u"Provide a single argument", out + err) + options = runner.Options() + options.wormhole = None + + rc, out, err = yield run_cli( + "-d", intro_dir, + "invite", + "--shares-needed", "1", + "--shares-happy", "1", + "--shares-total", "1", + options=options, + ) + self.assertTrue(rc) + self.assertIn(u"Provide a single argument", out + err) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py new file mode 100644 index 000000000..b60980bff --- /dev/null +++ b/src/allmydata/test/cli/wormholetesting.py @@ -0,0 +1,304 @@ +""" +An in-memory implementation of some of the magic-wormhole interfaces for +use by automated tests. + +For example:: + + async def peerA(mw): + wormhole = mw.create("myapp", "wss://myserver", reactor) + code = await wormhole.get_code() + print(f"I have a code: {code}") + message = await wormhole.when_received() + print(f"I have a message: {message}") + + async def local_peerB(helper, mw): + peerA_wormhole = await helper.wait_for_wormhole("myapp", "wss://myserver") + code = await peerA_wormhole.when_code() + + peerB_wormhole = mw.create("myapp", "wss://myserver") + peerB_wormhole.set_code(code) + + peerB_wormhole.send_message("Hello, peer A") + + # Run peerA against local_peerB with pure in-memory message passing. + server, helper = memory_server() + run(gather(peerA(server), local_peerB(helper, server))) + + # Run peerA against a peerB somewhere out in the world, using a real + # wormhole relay server somewhere. + import wormhole + run(peerA(wormhole)) +""" + +from __future__ import annotations + +from typing import Iterator +from collections.abc import Awaitable +from inspect import getargspec +from itertools import count +from sys import stderr + +from attrs import frozen, define, field, Factory +from twisted.internet.defer import Deferred, DeferredQueue, succeed +from wormhole._interfaces import IWormhole +from wormhole.wormhole import create +from zope.interface import implementer + + +@define +class MemoryWormholeServer(object): + """ + A factory for in-memory wormholes. + + :ivar _apps: Wormhole state arranged by the application id and relay URL + it belongs to. + + :ivar _waiters: Observers waiting for a wormhole to be created for a + specific application id and relay URL combination. + """ + _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + + def create( + self, + appid, + relay_url, + reactor, + versions={}, + delegate=None, + journal=None, + tor=None, + timing=None, + stderr=stderr, + _eventual_queue=None, + _enable_dilate=False, + ): + """ + Create a wormhole. It will be able to connect to other wormholes created + by this instance (and constrained by the normal appid/relay_url + rules). + """ + if tor is not None: + raise ValueError("Cannot deal with Tor right now.") + if _enable_dilate: + raise ValueError("Cannot deal with dilation right now.") + + key = (relay_url, appid) + wormhole = _MemoryWormhole(self._view(key)) + if key in self._waiters: + self._waiters.pop(key).callback(wormhole) + return wormhole + + def _view(self, key: tuple[str, str]) -> _WormholeServerView: + """ + Created a view onto this server's state that is limited by a certain + appid/relay_url pair. + """ + return _WormholeServerView(self, key) + + +@frozen +class TestingHelper(object): + """ + Provide extra functionality for interacting with an in-memory wormhole + implementation. + + This is intentionally a separate API so that it is not confused with + proper public interface of the real wormhole implementation. + """ + _server: MemoryWormholeServer + + async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + """ + Wait for a wormhole to appear at a specific location. + + :param appid: The appid that the resulting wormhole will have. + + :param relay_url: The URL of the relay at which the resulting wormhole + will presume to be created. + + :return: The first wormhole to be created which matches the given + parameters. + """ + key = relay_url, appid + if key in self._server._waiters: + raise ValueError(f"There is already a waiter for {key}") + d = Deferred() + self._server._waiters[key] = d + wormhole = await d + return wormhole + + +def _verify(): + """ + Roughly confirm that the in-memory wormhole creation function matches the + interface of the real implementation. + """ + # Poor man's interface verification. + + a = getargspec(create) + b = getargspec(MemoryWormholeServer.create) + # I know it has a `self` argument at the beginning. That's okay. + b = b._replace(args=b.args[1:]) + assert a == b, "{} != {}".format(a, b) + + +_verify() + + +@define +class _WormholeApp(object): + """ + Represent a collection of wormholes that belong to the same + appid/relay_url scope. + """ + wormholes: dict = field(default=Factory(dict)) + _waiting: dict = field(default=Factory(dict)) + _counter: Iterator[int] = field(default=Factory(count)) + + def allocate_code(self, wormhole, code): + """ + Allocate a new code for the given wormhole. + + This also associates the given wormhole with the code for future + lookup. + + Code generation logic is trivial and certainly not good enough for any + real use. It is sufficient for automated testing, though. + """ + if code is None: + code = "{}-persnickety-tardigrade".format(next(self._counter)) + self.wormholes.setdefault(code, []).append(wormhole) + try: + waiters = self._waiting.pop(code) + except KeyError: + pass + else: + for w in waiters: + w.callback(wormhole) + + return code + + def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + """ + Return a ``Deferred`` which fires with the next wormhole to be associated + with the given code. This is used to let the first end of a wormhole + rendezvous with the second end. + """ + d = Deferred() + self._waiting.setdefault(code, []).append(d) + return d + + +@frozen +class _WormholeServerView(object): + """ + Present an interface onto the server to be consumed by individual + wormholes. + """ + _server: MemoryWormholeServer + _key: tuple[str, str] + + def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + """ + Allocate a new code for the given wormhole in the scope associated with + this view. + """ + app = self._server._apps.setdefault(self._key, _WormholeApp()) + return app.allocate_code(wormhole, code) + + def wormhole_by_code(self, code, exclude): + """ + Retrieve all wormholes previously associated with a code. + """ + app = self._server._apps[self._key] + wormholes = app.wormholes[code] + try: + [wormhole] = list(wormhole for wormhole in wormholes if wormhole != exclude) + except ValueError: + return app.wait_for_wormhole(code) + return succeed(wormhole) + + +@implementer(IWormhole) +@define +class _MemoryWormhole(object): + """ + Represent one side of a wormhole as conceived by ``MemoryWormholeServer``. + """ + + _view: _WormholeServerView + _code: str = None + _payload: DeferredQueue = field(default=Factory(DeferredQueue)) + _waiting_for_code: list[Deferred] = field(default=Factory(list)) + _allocated: bool = False + + def allocate_code(self): + if self._code is not None: + raise ValueError( + "allocate_code used with a wormhole which already has a code" + ) + self._allocated = True + self._code = self._view.allocate_code(self, None) + waiters = self._waiting_for_code + self._waiting_for_code = None + for d in waiters: + d.callback(self._code) + + def set_code(self, code): + if self._code is None: + self._code = code + self._view.allocate_code(self, code) + else: + raise ValueError("set_code used with a wormhole which already has a code") + + def when_code(self): + if self._code is None: + d = Deferred() + self._waiting_for_code.append(d) + return d + return succeed(self._code) + + get_code = when_code + + def get_welcome(self): + return succeed("welcome") + + def send_message(self, payload): + self._payload.put(payload) + + def when_received(self): + if self._code is None: + raise ValueError( + "This implementation requires set_code or allocate_code " + "before when_received." + ) + d = self._view.wormhole_by_code(self._code, exclude=self) + + def got_wormhole(wormhole): + msg = wormhole._payload.get() + return msg + + d.addCallback(got_wormhole) + return d + + get_message = when_received + + def close(self): + pass + + # 0.9.2 compatibility + def get_code(self): + if self._code is None: + self.allocate_code() + return self.when_code() + + get = when_received + + +def memory_server() -> tuple[MemoryWormholeServer, TestingHelper]: + """ + Create a paired in-memory wormhole server and testing helper. + """ + server = MemoryWormholeServer() + return server, TestingHelper(server) From e35bab966351a7a24238430f25efb3dd24cdd89c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 11:01:35 -0400 Subject: [PATCH 04/12] news fragment --- newsfragments/3526.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3526.minor diff --git a/newsfragments/3526.minor b/newsfragments/3526.minor new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/newsfragments/3526.minor @@ -0,0 +1 @@ + From b0fffabed0aa525610714864ed74f3dd6c7445e8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:10:02 -0400 Subject: [PATCH 05/12] remove unnecessary module-scope wormhole used this during testing so the other mock() calls wouldn't explode in a boring way --- src/allmydata/scripts/tahoe_invite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index c5f08f588..b62d6a463 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -42,8 +42,6 @@ class InviteOptions(usage.Options): self['nick'] = args[0].strip() -wormhole = None - @defer.inlineCallbacks def _send_config_via_wormhole(options, config): out = options.stdout From 71b5cd9e0d64643a907d365c8d9580384cb92d4b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:13:48 -0400 Subject: [PATCH 06/12] rewrite comment annotations with syntax --- src/allmydata/test/cli/test_invite.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index c4bb6fd7e..50f446ae2 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -137,8 +137,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args=(), tahoe_config=None): - # type: (Sequence[bytes], Optional[bytes]) -> defer.Deferred + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. From 0f61a1dab9e72152e938ea3e5279ed1a1ac65d9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 12 Apr 2022 14:33:11 -0400 Subject: [PATCH 07/12] Factor some duplication out of the test methods --- src/allmydata/test/cli/test_invite.py | 149 ++++++++++++-------------- 1 file changed, 70 insertions(+), 79 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 50f446ae2..5b4871944 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -4,8 +4,9 @@ Tests for ``tahoe invite``. import json import os +from functools import partial from os.path import join -from typing import Optional, Sequence +from typing import Awaitable, Callable, Optional, Sequence, TypeVar from twisted.internet import defer from twisted.trial import unittest @@ -16,7 +17,58 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server + + +async def open_wormhole() -> tuple[Callable, IWormhole, str]: + """ + Create a new in-memory wormhole server, open one end of a wormhole, and + return it and related info. + + :return: A three-tuple allowing use of the wormhole. The first element is + a callable like ``run_cli`` but which will run commands so that they + use the in-memory wormhole server instead of a real one. The second + element is the open wormhole. The third element is the wormhole's + code. + """ + server = MemoryWormholeServer() + options = runner.Options() + options.wormhole = server + reactor = object() + + wormhole = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + code = await wormhole.get_code() + + return (partial(run_cli, options=options), wormhole, code) + + +def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: + """ + Send a list of message through a wormhole. + """ + for msg in messages: + wormhole.send_message(dumps_bytes(msg)) + + +A = TypeVar("A") +B = TypeVar("B") + +def concurrently( + client: Callable[[], Awaitable[A]], + server: Callable[[], Awaitable[B]], +) -> defer.Deferred[tuple[A, B]]: + """ + Run two asynchronous functions concurrently and asynchronously return a + tuple of both their results. + """ + return defer.gatherResults([ + defer.Deferred.fromCoroutine(client()), + defer.Deferred.fromCoroutine(server()), + ]) class Join(GridTestMixin, CLITestMixin, unittest.TestCase): @@ -33,18 +85,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): successfully join after an invite """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -53,15 +95,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"nickname": u"somethinghopefullyunique", u"introducer": u"pb://foo", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) self.assertEqual(0, rc) @@ -86,18 +125,8 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): Server sends JSON with unknown/illegal key """ node_dir = self.mktemp() - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v1": {}}}, { u"shares-needed": 1, @@ -107,15 +136,12 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): u"introducer": u"pb://foo", u"something-else": u"not allowed", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, node_dir, - options=options, ) # should still succeed -- just ignores the not-whitelisted @@ -137,7 +163,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): intro_dir, ) - async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[byte] = None) -> str: + async def _invite_success(self, extra_args: Sequence[bytes] = (), tahoe_config: Optional[bytes] = None) -> str: """ Exercise an expected-success case of ``tahoe invite``. @@ -217,10 +243,7 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): ) return invite - invite, _ = await defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + invite, _ = await concurrently(client, server) return invite @@ -340,10 +363,7 @@ shares.total = 6 # Send some surprising client abilities. other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -398,10 +418,7 @@ shares.total = 6 # Send a no-abilities message through to the server. other_end.send_message(dumps_bytes({})) - yield defer.gatherResults(map( - defer.Deferred.fromCoroutine, - [client(), server()], - )) + yield concurrently(client, server) @defer.inlineCallbacks @@ -416,18 +433,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - wormhole_server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = wormhole_server - reactor = object() - - wormhole = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {u"abilities": {u"server-v9000": {}}}, { "shares-needed": "1", @@ -436,15 +443,12 @@ shares.total = 6 "nickname": "foo", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "foo", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'server-v1' in server abilities", out + err) @@ -461,18 +465,8 @@ shares.total = 6 with open(join(priv_dir, "introducer.furl"), "w") as f: f.write("pb://fooblam\n") - server = MemoryWormholeServer() - options = runner.Options() - options.wormhole = server - reactor = object() - - wormhole = server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - code = yield wormhole.get_code() - messages = [ + run_cli, wormhole, code = yield defer.Deferred.fromCoroutine(open_wormhole()) + send_messages(wormhole, [ {}, { "shares-needed": "1", @@ -481,15 +475,12 @@ shares.total = 6 "nickname": "bar", "introducer": "pb://fooblam", }, - ] - for msg in messages: - wormhole.send_message(dumps_bytes(msg)) + ]) rc, out, err = yield run_cli( "create-client", "--join", code, "bar", - options=options, ) self.assertNotEqual(rc, 0) self.assertIn("Expected 'abilities' in server introduction", out + err) From 10f79ce8aa8ac07637ffcfc0af3a9003ff2736ba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:39:52 -0400 Subject: [PATCH 08/12] Use __future__.annotations in test_invite for generic builtins too --- src/allmydata/test/cli/test_invite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 5b4871944..7d2250bac 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -2,6 +2,8 @@ Tests for ``tahoe invite``. """ +from __future__ import annotations + import json import os from functools import partial From ec5be01f38ffdca34930b2301a206d1de1ecb047 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 14:50:38 -0400 Subject: [PATCH 09/12] more completely annotate types in the wormholetesting module --- src/allmydata/test/cli/wormholetesting.py | 51 ++++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index b60980bff..715e82236 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator +from typing import Iterator, Optional, Sequence from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -44,6 +44,11 @@ from wormhole._interfaces import IWormhole from wormhole.wormhole import create from zope.interface import implementer +WormholeCode = str +WormholeMessage = bytes +AppId = str +RelayURL = str +ApplicationKey = tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): @@ -56,8 +61,8 @@ class MemoryWormholeServer(object): :ivar _waiters: Observers waiting for a wormhole to be created for a specific application id and relay URL combination. """ - _apps: dict[tuple[str, str], _WormholeApp] = field(default=Factory(dict)) - _waiters: dict[tuple[str, str], Deferred] = field(default=Factory(dict)) + _apps: dict[ApplicationKey, _WormholeApp] = field(default=Factory(dict)) + _waiters: dict[ApplicationKey, Deferred] = field(default=Factory(dict)) def create( self, @@ -89,7 +94,7 @@ class MemoryWormholeServer(object): self._waiters.pop(key).callback(wormhole) return wormhole - def _view(self, key: tuple[str, str]) -> _WormholeServerView: + def _view(self, key: ApplicationKey) -> _WormholeServerView: """ Created a view onto this server's state that is limited by a certain appid/relay_url pair. @@ -108,7 +113,7 @@ class TestingHelper(object): """ _server: MemoryWormholeServer - async def wait_for_wormhole(self, appid: str, relay_url: str) -> IWormhole: + async def wait_for_wormhole(self, appid: AppId, relay_url: RelayURL) -> IWormhole: """ Wait for a wormhole to appear at a specific location. @@ -120,7 +125,7 @@ class TestingHelper(object): :return: The first wormhole to be created which matches the given parameters. """ - key = relay_url, appid + key = (relay_url, appid) if key in self._server._waiters: raise ValueError(f"There is already a waiter for {key}") d = Deferred() @@ -152,11 +157,11 @@ class _WormholeApp(object): Represent a collection of wormholes that belong to the same appid/relay_url scope. """ - wormholes: dict = field(default=Factory(dict)) - _waiting: dict = field(default=Factory(dict)) + wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) - def allocate_code(self, wormhole, code): + def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole. @@ -179,7 +184,7 @@ class _WormholeApp(object): return code - def wait_for_wormhole(self, code: str) -> Awaitable[_MemoryWormhole]: + def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]: """ Return a ``Deferred`` which fires with the next wormhole to be associated with the given code. This is used to let the first end of a wormhole @@ -197,9 +202,9 @@ class _WormholeServerView(object): wormholes. """ _server: MemoryWormholeServer - _key: tuple[str, str] + _key: ApplicationKey - def allocate_code(self, wormhole: _MemoryWormhole, code: str) -> str: + def allocate_code(self, wormhole: _MemoryWormhole, code: Optional[WormholeCode]) -> WormholeCode: """ Allocate a new code for the given wormhole in the scope associated with this view. @@ -207,7 +212,7 @@ class _WormholeServerView(object): app = self._server._apps.setdefault(self._key, _WormholeApp()) return app.allocate_code(wormhole, code) - def wormhole_by_code(self, code, exclude): + def wormhole_by_code(self, code: WormholeCode, exclude: object) -> Deferred[IWormhole]: """ Retrieve all wormholes previously associated with a code. """ @@ -228,46 +233,42 @@ class _MemoryWormhole(object): """ _view: _WormholeServerView - _code: str = None + _code: Optional[WormholeCode] = None _payload: DeferredQueue = field(default=Factory(DeferredQueue)) _waiting_for_code: list[Deferred] = field(default=Factory(list)) - _allocated: bool = False - def allocate_code(self): + def allocate_code(self) -> None: if self._code is not None: raise ValueError( "allocate_code used with a wormhole which already has a code" ) - self._allocated = True self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code self._waiting_for_code = None for d in waiters: d.callback(self._code) - def set_code(self, code): + def set_code(self, code: WormholeCode) -> None: if self._code is None: self._code = code self._view.allocate_code(self, code) else: raise ValueError("set_code used with a wormhole which already has a code") - def when_code(self): + def when_code(self) -> Deferred[WormholeCode]: if self._code is None: d = Deferred() self._waiting_for_code.append(d) return d return succeed(self._code) - get_code = when_code - def get_welcome(self): return succeed("welcome") - def send_message(self, payload): + def send_message(self, payload: WormholeMessage) -> None: self._payload.put(payload) - def when_received(self): + def when_received(self) -> Deferred[WormholeMessage]: if self._code is None: raise ValueError( "This implementation requires set_code or allocate_code " @@ -284,11 +285,11 @@ class _MemoryWormhole(object): get_message = when_received - def close(self): + def close(self) -> None: pass # 0.9.2 compatibility - def get_code(self): + def get_code(self) -> Deferred[WormholeCode]: if self._code is None: self.allocate_code() return self.when_code() From 38e1e93a75356a9275f7fc6b23bd998bc55d012e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 15:42:10 -0400 Subject: [PATCH 10/12] factor the duplicate client logic out --- src/allmydata/test/cli/test_invite.py | 157 +++++++++++--------------- 1 file changed, 69 insertions(+), 88 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 7d2250bac..9f2607433 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -8,7 +8,7 @@ import json import os from functools import partial from os.path import join -from typing import Awaitable, Callable, Optional, Sequence, TypeVar +from typing import Awaitable, Callable, Optional, Sequence, TypeVar, Union from twisted.internet import defer from twisted.trial import unittest @@ -21,6 +21,12 @@ from ..no_network import GridTestMixin from .common import CLITestMixin from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +# Logically: +# JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] +# +# But practically: +JSONable = Union[dict, None, int, float, str, list] + async def open_wormhole() -> tuple[Callable, IWormhole, str]: """ @@ -48,7 +54,42 @@ async def open_wormhole() -> tuple[Callable, IWormhole, str]: return (partial(run_cli, options=options), wormhole, code) -def send_messages(wormhole: IWormhole, messages: list[dict]) -> None: +def make_simple_peer( + reactor, + server: MemoryWormholeServer, + helper: TestingHelper, + messages: Sequence[JSONable], +) -> Callable[[], Awaitable[IWormhole]]: + """ + Make a wormhole peer that just sends the given messages. + + The returned function returns an awaitable that fires with the peer's end + of the wormhole. + """ + async def peer() -> IWormhole: + # Run the client side of the invitation by manually pumping a + # message through the wormhole. + + # First, wait for the server to create the wormhole at all. + wormhole = await helper.wait_for_wormhole( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + ) + # Then read out its code and open the other side of the wormhole. + code = await wormhole.when_code() + other_end = server.create( + "tahoe-lafs.org/invite", + "ws://wormhole.tahoe-lafs.org:4000/v1", + reactor, + ) + other_end.set_code(code) + send_messages(other_end, messages) + return other_end + + return peer + + +def send_messages(wormhole: IWormhole, messages: Sequence[JSONable]) -> None: """ Send a list of message through a wormhole. """ @@ -200,55 +241,34 @@ class Invite(GridTestMixin, CLITestMixin, unittest.TestCase): options=options, ) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. + # Send a proper client abilities message. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v1": {}}}]) + other_end, _ = await concurrently(client, server) - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a proper client abilities message. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v1": {}}})) - - # Check the server's messages. First, it should announce its - # abilities correctly. - server_abilities = json.loads(await other_end.when_received()) - self.assertEqual( - server_abilities, + # Check the server's messages. First, it should announce its + # abilities correctly. + server_abilities = json.loads(await other_end.when_received()) + self.assertEqual( + server_abilities, + { + "abilities": { - "abilities": - { - "server-v1": {} - }, + "server-v1": {} }, - ) + }, + ) - # Second, it should have an invitation with a nickname and - # introducer furl. - invite = json.loads(await other_end.when_received()) - self.assertEqual( - invite["nickname"], "foo", - ) - self.assertEqual( - invite["introducer"], "pb://fooblam", - ) - return invite - - invite, _ = await concurrently(client, server) + # Second, it should have an invitation with a nickname and introducer + # furl. + invite = json.loads(await other_end.when_received()) + self.assertEqual( + invite["nickname"], "foo", + ) + self.assertEqual( + invite["introducer"], "pb://fooblam", + ) return invite - @defer.inlineCallbacks def test_invite_success(self): """ @@ -344,30 +364,10 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'client-v1' in abilities", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send some surprising client abilities. - other_end.send_message(dumps_bytes({u"abilities": {u"client-v9000": {}}})) - + # Send some surprising client abilities. + client = make_simple_peer(reactor, wormhole_server, helper, [{u"abilities": {u"client-v9000": {}}}]) yield concurrently(client, server) - @defer.inlineCallbacks def test_invite_no_client_abilities(self): """ @@ -399,27 +399,8 @@ shares.total = 6 self.assertNotEqual(rc, 0) self.assertIn(u"No 'abilities' from client", out + err) - async def client(): - # Run the client side of the invitation by manually pumping a - # message through the wormhole. - - # First, wait for the server to create the wormhole at all. - wormhole = await helper.wait_for_wormhole( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - ) - # Then read out its code and open the other side of the wormhole. - code = await wormhole.when_code() - other_end = wormhole_server.create( - "tahoe-lafs.org/invite", - "ws://wormhole.tahoe-lafs.org:4000/v1", - reactor, - ) - other_end.set_code(code) - - # Send a no-abilities message through to the server. - other_end.send_message(dumps_bytes({})) - + # Send a no-abilities message through to the server. + client = make_simple_peer(reactor, wormhole_server, helper, [{}]) yield concurrently(client, server) From 03674bd4526ce2374f5077d50cd6da603e78f48a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 16:01:32 -0400 Subject: [PATCH 11/12] use Tuple for type alias __future__.annotations only fixes py37/generic builtins in annotations syntax, not arbitrary expressions --- src/allmydata/test/cli/wormholetesting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 715e82236..0cee78a5a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence +from typing import Iterator, Optional, Sequence, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -48,7 +48,7 @@ WormholeCode = str WormholeMessage = bytes AppId = str RelayURL = str -ApplicationKey = tuple[RelayURL, AppId] +ApplicationKey = Tuple[RelayURL, AppId] @define class MemoryWormholeServer(object): From f34e01649df46753cf4f129293b7b2fb2506a757 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 13 Apr 2022 18:35:18 -0400 Subject: [PATCH 12/12] some more fixes for mypy --- src/allmydata/test/cli/test_invite.py | 2 +- src/allmydata/test/cli/wormholetesting.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 9f2607433..07756eeed 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -19,7 +19,7 @@ from ...util.jsonbytes import dumps_bytes from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin -from .wormholetesting import IWormhole, MemoryWormholeServer, memory_server +from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server # Logically: # JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]] diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index 0cee78a5a..744f9d75a 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -32,7 +32,7 @@ For example:: from __future__ import annotations -from typing import Iterator, Optional, Sequence, Tuple +from typing import Iterator, Optional, List, Tuple from collections.abc import Awaitable from inspect import getargspec from itertools import count @@ -158,7 +158,7 @@ class _WormholeApp(object): appid/relay_url scope. """ wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict)) - _waiting: dict[WormholeCode, Sequence[Deferred]] = field(default=Factory(dict)) + _waiting: dict[WormholeCode, List[Deferred]] = field(default=Factory(dict)) _counter: Iterator[int] = field(default=Factory(count)) def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode: @@ -244,7 +244,7 @@ class _MemoryWormhole(object): ) self._code = self._view.allocate_code(self, None) waiters = self._waiting_for_code - self._waiting_for_code = None + self._waiting_for_code = [] for d in waiters: d.callback(self._code)