Make tahoe create-node use the new listener protocol

src/allmydata/scripts/create_node.py

@@ -1,3 +1,8 @@
+
+from __future__ import annotations
+
+from typing import Optional
+
 import io
 import os
 
@@ -21,7 +26,37 @@ from allmydata.scripts.common import (
 from allmydata.scripts.default_nodedir import _default_nodedir
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding
-from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json
+
+i2p_provider: Listener
+tor_provider: Listener
+
+from allmydata.util import fileutil, i2p_provider, tor_provider, jsonbytes as json
+
+from ..listeners import ListenerConfig, Listener, TCPProvider, StaticProvider
+
+def _get_listeners() -> dict[str, Listener]:
+    """
+    Get all of the kinds of listeners we might be able to use.
+    """
+    return {
+        "tor": tor_provider,
+        "i2p": i2p_provider,
+        "tcp": TCPProvider(),
+        "none": StaticProvider(
+            available=True,
+            hide_ip=False,
+            config=defer.succeed(None),
+            # This is supposed to be an IAddressFamily but we have none for
+            # this kind of provider.  We could implement new client and server
+            # endpoint types that always fail and pass an IAddressFamily here
+            # that uses those.  Nothing would ever even ask for them (at
+            # least, yet), let alone try to use them, so that's a lot of extra
+            # work for no practical result so I'm not doing it now.
+            address=None, # type: ignore[arg-type]
+        ),
+    }
+
+_LISTENERS = _get_listeners()
 
 dummy_tac = """
 import sys
@@ -98,8 +133,11 @@ def validate_where_options(o):
         if o['listen'] != "none" and o.get('join', None) is None:
             listeners = o['listen'].split(",")
             for l in listeners:
-                if l not in ["tcp", "tor", "i2p"]:
+                if l not in _LISTENERS:
-                    raise UsageError("--listen= must be none, or one/some of: tcp, tor, i2p")
+                    raise UsageError(
+                        "--listen= must be one/some of: "
+                        f"{', '.join(sorted(_LISTENERS))}",
+                    )
             if 'tcp' in listeners and not o['hostname']:
                 raise UsageError("--listen=tcp requires --hostname=")
             if 'tcp' not in listeners and o['hostname']:
@@ -108,7 +146,7 @@ def validate_where_options(o):
 def validate_tor_options(o):
     use_tor = "tor" in o["listen"].split(",")
     if use_tor or any((o["tor-launch"], o["tor-control-port"])):
-        if tor_provider._import_txtorcon() is None:
+        if not _LISTENERS["tor"].is_available():
             raise UsageError(
                 "Specifying any Tor options requires the 'txtorcon' module"
             )
@@ -123,7 +161,7 @@ def validate_tor_options(o):
 def validate_i2p_options(o):
     use_i2p = "i2p" in o["listen"].split(",")
     if use_i2p or any((o["i2p-launch"], o["i2p-sam-port"])):
-        if i2p_provider._import_txi2p() is None:
+        if not _LISTENERS["i2p"].is_available():
             raise UsageError(
                 "Specifying any I2P options requires the 'txi2p' module"
             )
@@ -145,7 +183,7 @@ class _CreateBaseOptions(BasedirOptions):
     def postOptions(self):
         super(_CreateBaseOptions, self).postOptions()
         if self['hide-ip']:
-            if tor_provider._import_txtorcon() is None and i2p_provider._import_txi2p() is None:
+            if not (_LISTENERS["tor"].is_available() or _LISTENERS["i2p"].is_available()):
                 raise UsageError(
                     "--hide-ip was specified but neither 'txtorcon' nor 'txi2p' "
                     "are installed.\nTo do so:\n   pip install tahoe-lafs[tor]\nor\n"
@@ -218,8 +256,20 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions):
         validate_i2p_options(self)
 
 
-@defer.inlineCallbacks
+def merge_config(
-def write_node_config(c, config):
+        left: Optional[ListenerConfig],
+        right: Optional[ListenerConfig],
+) -> Optional[ListenerConfig]:
+    if left is None or right is None:
+        return None
+    return ListenerConfig(
+        list(left.tub_ports) + list(right.tub_ports),
+        list(left.tub_locations) + list(right.tub_locations),
+        dict(list(left.node_config.items()) + list(right.node_config.items())),
+    )
+
+
+async def write_node_config(c, config):
     # this is shared between clients and introducers
     c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c))
     c.write("\n")
@@ -232,9 +282,10 @@ def write_node_config(c, config):
 
     if config["hide-ip"]:
         c.write("[connections]\n")
-        if tor_provider._import_txtorcon():
+        if _LISTENERS["tor"].is_available():
             c.write("tcp = tor\n")
         else:
+            # XXX What about i2p?
             c.write("tcp = disabled\n")
         c.write("\n")
 
@@ -253,38 +304,23 @@ def write_node_config(c, config):
     c.write("web.port = %s\n" % (webport,))
     c.write("web.static = public_html\n")
 
-    listeners = config['listen'].split(",")
+    listener_config = ListenerConfig([], [], {})
+    for listener_name in config['listen'].split(","):
+        listener = _LISTENERS[listener_name]
+        listener_config = merge_config(
+            (await listener.create_config(reactor, config)),
+            listener_config,
+        )
 
-    tor_config = {}
+    if listener_config is None:
-    i2p_config = {}
+        tub_ports = ["disabled"]
-    tub_ports = []
+        tub_locations = ["disabled"]
-    tub_locations = []
-    if listeners == ["none"]:
-        c.write("tub.port = disabled\n")
-        c.write("tub.location = disabled\n")
     else:
-        if "tor" in listeners:
+        tub_ports = listener_config.tub_ports
-            (tor_config, tor_port, tor_location) = \
+        tub_locations = listener_config.tub_locations
-                         yield tor_provider.create_config(reactor, config)
+
-            tub_ports.append(tor_port)
+    c.write("tub.port = %s\n" % ",".join(tub_ports))
-            tub_locations.append(tor_location)
+    c.write("tub.location = %s\n" % ",".join(tub_locations))
-        if "i2p" in listeners:
-            (i2p_config, i2p_port, i2p_location) = \
-                         yield i2p_provider.create_config(reactor, config)
-            tub_ports.append(i2p_port)
-            tub_locations.append(i2p_location)
-        if "tcp" in listeners:
-            if config["port"]: # --port/--location are a pair
-                tub_ports.append(config["port"])
-                tub_locations.append(config["location"])
-            else:
-                assert "hostname" in config
-                hostname = config["hostname"]
-                new_port = iputil.allocate_tcp_port()
-                tub_ports.append("tcp:%s" % new_port)
-                tub_locations.append("tcp:%s:%s" % (hostname, new_port))
-        c.write("tub.port = %s\n" % ",".join(tub_ports))
-        c.write("tub.location = %s\n" % ",".join(tub_locations))
     c.write("\n")
 
     c.write("#log_gatherer.furl =\n")
@@ -294,17 +330,12 @@ def write_node_config(c, config):
     c.write("#ssh.authorized_keys_file = ~/.ssh/authorized_keys\n")
     c.write("\n")
 
-    if tor_config:
+    if listener_config is not None:
-        c.write("[tor]\n")
+        for section, items in listener_config.node_config.items():
-        for key, value in list(tor_config.items()):
+            c.write(f"[{section}]\n")
-            c.write("%s = %s\n" % (key, value))
+            for k, v in items:
-        c.write("\n")
+                c.write(f"{k} = {v}\n")
-
+            c.write("\n")
-    if i2p_config:
-        c.write("[i2p]\n")
-        for key, value in list(i2p_config.items()):
-            c.write("%s = %s\n" % (key, value))
-        c.write("\n")
 
 
 def write_client_config(c, config):
@@ -445,7 +476,7 @@ def create_node(config):
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
     cfg_name = os.path.join(basedir, "tahoe.cfg")
     with io.open(cfg_name, "w", encoding='utf-8') as c:
-        yield write_node_config(c, config)
+        yield defer.Deferred.fromCoroutine(write_node_config(c, config))
         write_client_config(c, config)
 
     print("Node created in %s" % quote_local_unicode_path(basedir), file=out)
@@ -488,7 +519,7 @@ def create_introducer(config):
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
     cfg_name = os.path.join(basedir, "tahoe.cfg")
     with io.open(cfg_name, "w", encoding='utf-8') as c:
-        yield write_node_config(c, config)
+        yield defer.Deferred.fromCoroutine(write_node_config(c, config))
 
     print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)
     defer.returnValue(0)

src/allmydata/test/cli/test_create.py

@@ -17,6 +17,7 @@ from ..common import (
     disable_modules,
 )
 from ...scripts import create_node
+from ...listeners import ListenerConfig, StaticProvider
 from ... import client
 
 def read_config(basedir):
@@ -45,7 +46,7 @@ class Config(unittest.TestCase):
             e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
             self.assertIn("option %s not recognized" % (option,), str(e))
 
-    def test_create_client_config(self):
+    async def test_create_client_config(self):
         d = self.mktemp()
         os.mkdir(d)
         fname = os.path.join(d, 'tahoe.cfg')
@@ -59,7 +60,7 @@ class Config(unittest.TestCase):
                     "shares-happy": "1",
                     "shares-total": "1",
                     }
-            create_node.write_node_config(f, opts)
+            await create_node.write_node_config(f, opts)
             create_node.write_client_config(f, opts)
 
         # should succeed, no exceptions
@@ -245,7 +246,7 @@ class Config(unittest.TestCase):
                               parse_cli,
                               "create-node", "--listen=tcp,none",
                               basedir)
-        self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p")
+        self.assertEqual(str(e), "--listen=tcp requires --hostname=")
 
     def test_node_listen_bad(self):
         basedir = self.mktemp()
@@ -253,7 +254,7 @@ class Config(unittest.TestCase):
                               parse_cli,
                               "create-node", "--listen=XYZZY,tcp",
                               basedir)
-        self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p")
+        self.assertEqual(str(e), "--listen= must be one/some of: i2p, none, tcp, tor")
 
     def test_node_listen_tor_hostname(self):
         e = self.assertRaises(usage.UsageError,
@@ -287,24 +288,15 @@ class Config(unittest.TestCase):
         self.assertIn("To avoid clobbering anything, I am going to quit now", err)
 
     @defer.inlineCallbacks
-    def test_node_slow_tor(self):
+    def test_node_slow(self):
-        basedir = self.mktemp()
         d = defer.Deferred()
-        self.patch(tor_provider, "create_config", lambda *a, **kw: d)
+        slow = StaticProvider(True, False, d, None)
-        d2 = run_cli("create-node", "--listen=tor", basedir)
+        create_node._LISTENERS["xxyzy"] = slow
-        d.callback(({}, "port", "location"))
+        self.addCleanup(lambda: create_node._LISTENERS.pop("xxyzy"))
-        rc, out, err = yield d2
-        self.assertEqual(rc, 0)
-        self.assertIn("Node created", out)
-        self.assertEqual(err, "")
 
-    @defer.inlineCallbacks
-    def test_node_slow_i2p(self):
         basedir = self.mktemp()
-        d = defer.Deferred()
+        d2 = run_cli("create-node", "--listen=xxyzy", basedir)
-        self.patch(i2p_provider, "create_config", lambda *a, **kw: d)
+        d.callback(None)
-        d2 = run_cli("create-node", "--listen=i2p", basedir)
-        d.callback(({}, "port", "location"))
         rc, out, err = yield d2
         self.assertEqual(rc, 0)
         self.assertIn("Node created", out)
@@ -369,10 +361,12 @@ def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[t
 class Tor(unittest.TestCase):
     def test_default(self):
         basedir = self.mktemp()
-        tor_config = {"abc": "def"}
+        tor_config = {"tor": [("abc", "def")]}
         tor_port = "ghi"
         tor_location = "jkl"
-        config_d = defer.succeed( (tor_config, tor_port, tor_location) )
+        config_d = defer.succeed(
+            ListenerConfig([tor_port], [tor_location], tor_config)
+        )
 
         calls = fake_config(self, tor_provider, config_d)
         rc, out, err = self.successResultOf(
@@ -391,10 +385,7 @@ class Tor(unittest.TestCase):
 
     def test_launch(self):
         basedir = self.mktemp()
-        tor_config = {"abc": "def"}
+        config_d = defer.succeed(None)
-        tor_port = "ghi"
-        tor_location = "jkl"
-        config_d = defer.succeed( (tor_config, tor_port, tor_location) )
 
         calls = fake_config(self, tor_provider, config_d)
         rc, out, err = self.successResultOf(
@@ -410,10 +401,7 @@ class Tor(unittest.TestCase):
 
     def test_control_port(self):
         basedir = self.mktemp()
-        tor_config = {"abc": "def"}
+        config_d = defer.succeed(None)
-        tor_port = "ghi"
-        tor_location = "jkl"
-        config_d = defer.succeed( (tor_config, tor_port, tor_location) )
 
         calls = fake_config(self, tor_provider, config_d)
         rc, out, err = self.successResultOf(
@@ -451,10 +439,10 @@ class Tor(unittest.TestCase):
 class I2P(unittest.TestCase):
     def test_default(self):
         basedir = self.mktemp()
-        i2p_config = {"abc": "def"}
+        i2p_config = {"i2p": [("abc", "def")]}
         i2p_port = "ghi"
         i2p_location = "jkl"
-        dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) )
+        dest_d = defer.succeed(ListenerConfig([i2p_port], [i2p_location], i2p_config))
 
         calls = fake_config(self, i2p_provider, dest_d)
         rc, out, err = self.successResultOf(
@@ -479,10 +467,7 @@ class I2P(unittest.TestCase):
 
     def test_sam_port(self):
         basedir = self.mktemp()
-        i2p_config = {"abc": "def"}
+        dest_d = defer.succeed(None)
-        i2p_port = "ghi"
-        i2p_location = "jkl"
-        dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) )
 
         calls = fake_config(self, i2p_provider, dest_d)
         rc, out, err = self.successResultOf(