ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 21:17:54 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1283 from exarkun/clean-up-tor-and-i2p-providers

Browse Source 
Abstract over some configuration manipulation done by `write_node_config` in `create_node.py`

Fixes: ticket:4004
This commit is contained in:
Jean-Paul Calderone 2023-07-21 09:31:05 -04:00 committed by GitHub
commit 8b8903c44f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 456 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mypy.ini
View File

@ -9,7 +9,7 @@ no_implicit_optional = True
warn_redundant_casts = True
strict_equality = True
[mypy-allmydata.test.cli.wormholetesting,allmydata.test.test_connection_status]
[mypy-allmydata.test.cli.wormholetesting,allmydata.listeners,allmydata.test.test_connection_status]
disallow_any_generics = True
disallow_subclassing_any = True
disallow_untyped_calls = True

0
newsfragments/4004.minor Normal file
View File

121
src/allmydata/listeners.py Normal file
View File

@ -0,0 +1,121 @@
"""
Define a protocol for listening on a transport such that Tahoe-LAFS can
communicate over it, manage configuration for it in its configuration file,
detect when it is possible to use it, etc.
"""
from __future__ import annotations
from typing import Any, Protocol, Sequence, Mapping, Optional, Union, Awaitable
from typing_extensions import Literal
from attrs import frozen
from twisted.python.usage import Options
from .interfaces import IAddressFamily
from .util.iputil import allocate_tcp_port
from .node import _Config
@frozen
class ListenerConfig:
"""
:ivar tub_ports: Entries to merge into ``[node]tub.port``.
:ivar tub_locations: Entries to merge into ``[node]tub.location``.
:ivar node_config: Entries to add into the overall Tahoe-LAFS
configuration beneath a section named after this listener.
"""
tub_ports: Sequence[str]
tub_locations: Sequence[str]
node_config: Mapping[str, Sequence[tuple[str, str]]]
class Listener(Protocol):
"""
An object which can listen on a transport and allow Tahoe-LAFS
communication to happen over it.
"""
def is_available(self) -> bool:
"""
Can this type of listener actually be used in this runtime
environment?
"""
def can_hide_ip(self) -> bool:
"""
Can the transport supported by this type of listener conceal the
node's public internet address from peers?
"""
async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]:
"""
Set up an instance of this listener according to the given
configuration parameters.
This may also allocate ephemeral resources if necessary.
:return: The created configuration which can be merged into the
overall *tahoe.cfg* configuration file.
"""
def create(self, reactor: Any, config: _Config) -> IAddressFamily:
"""
Instantiate this listener according to the given
previously-generated configuration.
:return: A handle on the listener which can be used to integrate it
into the Tahoe-LAFS node.
"""
class TCPProvider:
"""
Support plain TCP connections.
"""
def is_available(self) -> Literal[True]:
return True
def can_hide_ip(self) -> Literal[False]:
return False
async def create_config(self, reactor: Any, cli_config: Options) -> ListenerConfig:
tub_ports = []
tub_locations = []
if cli_config["port"]: # --port/--location are a pair
tub_ports.append(cli_config["port"])
tub_locations.append(cli_config["location"])
else:
assert "hostname" in cli_config
hostname = cli_config["hostname"]
new_port = allocate_tcp_port()
tub_ports.append(f"tcp:{new_port}")
tub_locations.append(f"tcp:{hostname}:{new_port}")
return ListenerConfig(tub_ports, tub_locations, {})
def create(self, reactor: Any, config: _Config) -> IAddressFamily:
raise NotImplementedError()
@frozen
class StaticProvider:
"""
A provider that uses all pre-computed values.
"""
_available: bool
_hide_ip: bool
_config: Union[Awaitable[Optional[ListenerConfig]], Optional[ListenerConfig]]
_address: IAddressFamily
def is_available(self) -> bool:
return self._available
def can_hide_ip(self) -> bool:
return self._hide_ip
async def create_config(self, reactor: Any, cli_config: Options) -> Optional[ListenerConfig]:
if self._config is None or isinstance(self._config, ListenerConfig):
return self._config
return await self._config
def create(self, reactor: Any, config: _Config) -> IAddressFamily:
return self._address

162
src/allmydata/scripts/create_node.py
View File

@ -1,3 +1,8 @@
from __future__ import annotations
from typing import Optional
import io
import os
@ -19,9 +24,40 @@ from allmydata.scripts.common import (
write_introducer,
)
from allmydata.scripts.default_nodedir import _default_nodedir
from allmydata.util import dictutil
from allmydata.util.assertutil import precondition
from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding
from allmydata.util import fileutil, i2p_provider, iputil, tor_provider, jsonbytes as json
i2p_provider: Listener
tor_provider: Listener
from allmydata.util import fileutil, i2p_provider, tor_provider, jsonbytes as json
from ..listeners import ListenerConfig, Listener, TCPProvider, StaticProvider
def _get_listeners() -> dict[str, Listener]:
"""
Get all of the kinds of listeners we might be able to use.
"""
return {
"tor": tor_provider,
"i2p": i2p_provider,
"tcp": TCPProvider(),
"none": StaticProvider(
available=True,
hide_ip=False,
config=defer.succeed(None),
# This is supposed to be an IAddressFamily but we have none for
# this kind of provider. We could implement new client and server
# endpoint types that always fail and pass an IAddressFamily here
# that uses those. Nothing would ever even ask for them (at
# least, yet), let alone try to use them, so that's a lot of extra
# work for no practical result so I'm not doing it now.
address=None, # type: ignore[arg-type]
),
}
_LISTENERS = _get_listeners()
dummy_tac = """
import sys
@ -98,8 +134,11 @@ def validate_where_options(o):
if o['listen'] != "none" and o.get('join', None) is None:
listeners = o['listen'].split(",")
for l in listeners:
if l not in ["tcp", "tor", "i2p"]:
raise UsageError("--listen= must be none, or one/some of: tcp, tor, i2p")
if l not in _LISTENERS:
raise UsageError(
"--listen= must be one/some of: "
f"{', '.join(sorted(_LISTENERS))}",
)
if 'tcp' in listeners and not o['hostname']:
raise UsageError("--listen=tcp requires --hostname=")
if 'tcp' not in listeners and o['hostname']:
@ -108,7 +147,7 @@ def validate_where_options(o):
def validate_tor_options(o):
use_tor = "tor" in o["listen"].split(",")
if use_tor or any((o["tor-launch"], o["tor-control-port"])):
if tor_provider._import_txtorcon() is None:
if not _LISTENERS["tor"].is_available():
raise UsageError(
"Specifying any Tor options requires the 'txtorcon' module"
)
@ -123,7 +162,7 @@ def validate_tor_options(o):
def validate_i2p_options(o):
use_i2p = "i2p" in o["listen"].split(",")
if use_i2p or any((o["i2p-launch"], o["i2p-sam-port"])):
if i2p_provider._import_txi2p() is None:
if not _LISTENERS["i2p"].is_available():
raise UsageError(
"Specifying any I2P options requires the 'txi2p' module"
)
@ -145,11 +184,17 @@ class _CreateBaseOptions(BasedirOptions):
def postOptions(self):
super(_CreateBaseOptions, self).postOptions()
if self['hide-ip']:
if tor_provider._import_txtorcon() is None and i2p_provider._import_txi2p() is None:
ip_hiders = dictutil.filter(lambda v: v.can_hide_ip(), _LISTENERS)
available = dictutil.filter(lambda v: v.is_available(), ip_hiders)
if not available:
raise UsageError(
"--hide-ip was specified but neither 'txtorcon' nor 'txi2p' "
"are installed.\nTo do so:\n pip install tahoe-lafs[tor]\nor\n"
" pip install tahoe-lafs[i2p]"
"--hide-ip was specified but no IP-hiding listener is installed.\n"
"Try one of these:\n" +
"".join([
f"\tpip install tahoe-lafs[{name}]\n"
for name
in ip_hiders
])
)
class CreateClientOptions(_CreateBaseOptions):
@ -218,8 +263,34 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions):
validate_i2p_options(self)
@defer.inlineCallbacks
def write_node_config(c, config):
def merge_config(
left: Optional[ListenerConfig],
right: Optional[ListenerConfig],
) -> Optional[ListenerConfig]:
"""
Merge two listener configurations into one configuration representing
both of them.
If either is ``None`` then the result is ``None``. This supports the
"disable listeners" functionality.
:raise ValueError: If the keys in the node configs overlap.
"""
if left is None or right is None:
return None
overlap = set(left.node_config) & set(right.node_config)
if overlap:
raise ValueError(f"Node configs overlap: {overlap}")
return ListenerConfig(
list(left.tub_ports) + list(right.tub_ports),
list(left.tub_locations) + list(right.tub_locations),
dict(list(left.node_config.items()) + list(right.node_config.items())),
)
async def write_node_config(c, config):
# this is shared between clients and introducers
c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c))
c.write("\n")
@ -232,9 +303,10 @@ def write_node_config(c, config):
if config["hide-ip"]:
c.write("[connections]\n")
if tor_provider._import_txtorcon():
if _LISTENERS["tor"].is_available():
c.write("tcp = tor\n")
else:
# XXX What about i2p?
c.write("tcp = disabled\n")
c.write("\n")
@ -253,38 +325,23 @@ def write_node_config(c, config):
c.write("web.port = %s\n" % (webport,))
c.write("web.static = public_html\n")
listeners = config['listen'].split(",")
listener_config = ListenerConfig([], [], {})
for listener_name in config['listen'].split(","):
listener = _LISTENERS[listener_name]
listener_config = merge_config(
(await listener.create_config(reactor, config)),
listener_config,
)
tor_config = {}
i2p_config = {}
tub_ports = []
tub_locations = []
if listeners == ["none"]:
c.write("tub.port = disabled\n")
c.write("tub.location = disabled\n")
if listener_config is None:
tub_ports = ["disabled"]
tub_locations = ["disabled"]
else:
if "tor" in listeners:
(tor_config, tor_port, tor_location) = \
yield tor_provider.create_config(reactor, config)
tub_ports.append(tor_port)
tub_locations.append(tor_location)
if "i2p" in listeners:
(i2p_config, i2p_port, i2p_location) = \
yield i2p_provider.create_config(reactor, config)
tub_ports.append(i2p_port)
tub_locations.append(i2p_location)
if "tcp" in listeners:
if config["port"]: # --port/--location are a pair
tub_ports.append(config["port"])
tub_locations.append(config["location"])
else:
assert "hostname" in config
hostname = config["hostname"]
new_port = iputil.allocate_tcp_port()
tub_ports.append("tcp:%s" % new_port)
tub_locations.append("tcp:%s:%s" % (hostname, new_port))
c.write("tub.port = %s\n" % ",".join(tub_ports))
c.write("tub.location = %s\n" % ",".join(tub_locations))
tub_ports = listener_config.tub_ports
tub_locations = listener_config.tub_locations
c.write("tub.port = %s\n" % ",".join(tub_ports))
c.write("tub.location = %s\n" % ",".join(tub_locations))
c.write("\n")
c.write("#log_gatherer.furl =\n")
@ -294,17 +351,12 @@ def write_node_config(c, config):
c.write("#ssh.authorized_keys_file = ~/.ssh/authorized_keys\n")
c.write("\n")
if tor_config:
c.write("[tor]\n")
for key, value in list(tor_config.items()):
c.write("%s = %s\n" % (key, value))
c.write("\n")
if i2p_config:
c.write("[i2p]\n")
for key, value in list(i2p_config.items()):
c.write("%s = %s\n" % (key, value))
c.write("\n")
if listener_config is not None:
for section, items in listener_config.node_config.items():
c.write(f"[{section}]\n")
for k, v in items:
c.write(f"{k} = {v}\n")
c.write("\n")
def write_client_config(c, config):
@ -445,7 +497,7 @@ def create_node(config):
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
cfg_name = os.path.join(basedir, "tahoe.cfg")
with io.open(cfg_name, "w", encoding='utf-8') as c:
yield write_node_config(c, config)
yield defer.Deferred.fromCoroutine(write_node_config(c, config))
write_client_config(c, config)
print("Node created in %s" % quote_local_unicode_path(basedir), file=out)
@ -488,7 +540,7 @@ def create_introducer(config):
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
cfg_name = os.path.join(basedir, "tahoe.cfg")
with io.open(cfg_name, "w", encoding='utf-8') as c:
yield write_node_config(c, config)
yield defer.Deferred.fromCoroutine(write_node_config(c, config))
print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)
defer.returnValue(0)

136
src/allmydata/test/cli/test_create.py
View File

@ -17,6 +17,7 @@ from ..common import (
disable_modules,
)
from ...scripts import create_node
from ...listeners import ListenerConfig, StaticProvider
from ... import client
def read_config(basedir):
@ -24,6 +25,68 @@ def read_config(basedir):
config = configutil.get_config(tahoe_cfg)
return config
class MergeConfigTests(unittest.TestCase):
"""
Tests for ``create_node.merge_config``.
"""
def test_disable_left(self) -> None:
"""
If the left argument to ``create_node.merge_config`` is ``None``
then the return value is ``None``.
"""
conf = ListenerConfig([], [], {})
self.assertEqual(None, create_node.merge_config(None, conf))
def test_disable_right(self) -> None:
"""
If the right argument to ``create_node.merge_config`` is ``None``
then the return value is ``None``.
"""
conf = ListenerConfig([], [], {})
self.assertEqual(None, create_node.merge_config(conf, None))
def test_disable_both(self) -> None:
"""
If both arguments to ``create_node.merge_config`` are ``None``
then the return value is ``None``.
"""
self.assertEqual(None, create_node.merge_config(None, None))
def test_overlapping_keys(self) -> None:
"""
If there are any keys in the ``node_config`` of the left and right
parameters that are shared then ``ValueError`` is raised.
"""
left = ListenerConfig([], [], {"foo": [("b", "ar")]})
right = ListenerConfig([], [], {"foo": [("ba", "z")]})
self.assertRaises(ValueError, lambda: create_node.merge_config(left, right))
def test_merge(self) -> None:
"""
``create_node.merge_config`` returns a ``ListenerConfig`` that has
all of the ports, locations, and node config from each of the two
``ListenerConfig`` values given.
"""
left = ListenerConfig(
["left-port"],
["left-location"],
{"left": [("f", "oo")]},
)
right = ListenerConfig(
["right-port"],
["right-location"],
{"right": [("ba", "r")]},
)
result = create_node.merge_config(left, right)
self.assertEqual(
ListenerConfig(
["left-port", "right-port"],
["left-location", "right-location"],
{"left": [("f", "oo")], "right": [("ba", "r")]},
),
result,
)
class Config(unittest.TestCase):
def test_client_unrecognized_options(self):
tests = [
@ -45,7 +108,14 @@ class Config(unittest.TestCase):
e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
self.assertIn("option %s not recognized" % (option,), str(e))
def test_create_client_config(self):
async def test_create_client_config(self):
"""
``create_node.write_client_config`` writes a configuration file
that can be parsed.
TODO Maybe we should test that we can recover the given configuration
from the parse, too.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
@ -59,7 +129,7 @@ class Config(unittest.TestCase):
"shares-happy": "1",
"shares-total": "1",
}
create_node.write_node_config(f, opts)
await create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
# should succeed, no exceptions
@ -245,7 +315,7 @@ class Config(unittest.TestCase):
parse_cli,
"create-node", "--listen=tcp,none",
basedir)
self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p")
self.assertEqual(str(e), "--listen=tcp requires --hostname=")
def test_node_listen_bad(self):
basedir = self.mktemp()
@ -253,7 +323,7 @@ class Config(unittest.TestCase):
parse_cli,
"create-node", "--listen=XYZZY,tcp",
basedir)
self.assertEqual(str(e), "--listen= must be none, or one/some of: tcp, tor, i2p")
self.assertEqual(str(e), "--listen= must be one/some of: i2p, none, tcp, tor")
def test_node_listen_tor_hostname(self):
e = self.assertRaises(usage.UsageError,
@ -287,24 +357,19 @@ class Config(unittest.TestCase):
self.assertIn("To avoid clobbering anything, I am going to quit now", err)
@defer.inlineCallbacks
def test_node_slow_tor(self):
basedir = self.mktemp()
def test_node_slow(self):
"""
A node can be created using a listener type that returns an
unfired Deferred from its ``create_config`` method.
"""
d = defer.Deferred()
self.patch(tor_provider, "create_config", lambda *a, **kw: d)
d2 = run_cli("create-node", "--listen=tor", basedir)
d.callback(({}, "port", "location"))
rc, out, err = yield d2
self.assertEqual(rc, 0)
self.assertIn("Node created", out)
self.assertEqual(err, "")
slow = StaticProvider(True, False, d, None)
create_node._LISTENERS["xxyzy"] = slow
self.addCleanup(lambda: create_node._LISTENERS.pop("xxyzy"))
@defer.inlineCallbacks
def test_node_slow_i2p(self):
basedir = self.mktemp()
d = defer.Deferred()
self.patch(i2p_provider, "create_config", lambda *a, **kw: d)
d2 = run_cli("create-node", "--listen=i2p", basedir)
d.callback(({}, "port", "location"))
d2 = run_cli("create-node", "--listen=xxyzy", basedir)
d.callback(None)
rc, out, err = yield d2
self.assertEqual(rc, 0)
self.assertIn("Node created", out)
@ -369,10 +434,12 @@ def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[t
class Tor(unittest.TestCase):
def test_default(self):
basedir = self.mktemp()
tor_config = {"abc": "def"}
tor_config = {"tor": [("abc", "def")]}
tor_port = "ghi"
tor_location = "jkl"
config_d = defer.succeed( (tor_config, tor_port, tor_location) )
config_d = defer.succeed(
ListenerConfig([tor_port], [tor_location], tor_config)
)
calls = fake_config(self, tor_provider, config_d)
rc, out, err = self.successResultOf(
@ -390,11 +457,12 @@ class Tor(unittest.TestCase):
self.assertEqual(cfg.get("node", "tub.location"), "jkl")
def test_launch(self):
"""
The ``--tor-launch`` command line option sets ``tor-launch`` to
``True``.
"""
basedir = self.mktemp()
tor_config = {"abc": "def"}
tor_port = "ghi"
tor_location = "jkl"
config_d = defer.succeed( (tor_config, tor_port, tor_location) )
config_d = defer.succeed(None)
calls = fake_config(self, tor_provider, config_d)
rc, out, err = self.successResultOf(
@ -409,11 +477,12 @@ class Tor(unittest.TestCase):
self.assertEqual(args[1]["tor-control-port"], None)
def test_control_port(self):
"""
The ``--tor-control-port`` command line parameter's value is
passed along as the ``tor-control-port`` value.
"""
basedir = self.mktemp()
tor_config = {"abc": "def"}
tor_port = "ghi"
tor_location = "jkl"
config_d = defer.succeed( (tor_config, tor_port, tor_location) )
config_d = defer.succeed(None)
calls = fake_config(self, tor_provider, config_d)
rc, out, err = self.successResultOf(
@ -451,10 +520,10 @@ class Tor(unittest.TestCase):
class I2P(unittest.TestCase):
def test_default(self):
basedir = self.mktemp()
i2p_config = {"abc": "def"}
i2p_config = {"i2p": [("abc", "def")]}
i2p_port = "ghi"
i2p_location = "jkl"
dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) )
dest_d = defer.succeed(ListenerConfig([i2p_port], [i2p_location], i2p_config))
calls = fake_config(self, i2p_provider, dest_d)
rc, out, err = self.successResultOf(
@ -479,10 +548,7 @@ class I2P(unittest.TestCase):
def test_sam_port(self):
basedir = self.mktemp()
i2p_config = {"abc": "def"}
i2p_port = "ghi"
i2p_location = "jkl"
dest_d = defer.succeed( (i2p_config, i2p_port, i2p_location) )
dest_d = defer.succeed(None)
calls = fake_config(self, i2p_provider, dest_d)
rc, out, err = self.successResultOf(

25
src/allmydata/test/test_dictutil.py
View File

@ -1,17 +1,9 @@
"""
Tests for allmydata.util.dictutil.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2, PY3
if PY2:
# dict omitted to match dictutil.py.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
from unittest import skipIf
@ -168,3 +160,18 @@ class TypedKeyDictPython2(unittest.TestCase):
# Demonstration of how bytes and unicode can be mixed:
d = {u"abc": 1}
self.assertEqual(d[b"abc"], 1)
class FilterTests(unittest.TestCase):
"""
Tests for ``dictutil.filter``.
"""
def test_filter(self) -> None:
"""
``dictutil.filter`` returns a ``dict`` that contains the key/value
pairs for which the value is matched by the given predicate.
"""
self.assertEqual(
{1: 2},
dictutil.filter(lambda v: v == 2, {1: 2, 2: 3}),
)

8
src/allmydata/test/test_i2p_provider.py
View File

@ -177,7 +177,7 @@ class CreateDest(unittest.TestCase):
with mock.patch("allmydata.util.i2p_provider.clientFromString",
return_value=ep) as cfs:
d = i2p_provider.create_config(reactor, cli_config)
tahoe_config_i2p, i2p_port, i2p_location = self.successResultOf(d)
i2p_config = self.successResultOf(d)
connect_to_i2p.assert_called_with(reactor, cli_config, txi2p)
cfs.assert_called_with(reactor, "goodport")
@ -189,9 +189,9 @@ class CreateDest(unittest.TestCase):
"dest.private_key_file": os.path.join("private",
"i2p_dest.privkey"),
}
self.assertEqual(tahoe_config_i2p, expected)
self.assertEqual(i2p_port, "listen:i2p")
self.assertEqual(i2p_location, "i2p:FOOBAR.b32.i2p:3457")
self.assertEqual(dict(i2p_config.node_config["i2p"]), expected)
self.assertEqual(i2p_config.tub_ports, ["listen:i2p"])
self.assertEqual(i2p_config.tub_locations, ["i2p:FOOBAR.b32.i2p:3457"])
_None = object()
class FakeConfig(dict):

20
src/allmydata/test/test_tor_provider.py
View File

@ -197,7 +197,7 @@ class CreateOnion(unittest.TestCase):
with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
return_value=999999):
d = tor_provider.create_config(reactor, cli_config)
tahoe_config_tor, tor_port, tor_location = self.successResultOf(d)
tor_config = self.successResultOf(d)
launch_tor.assert_called_with(reactor, executable,
os.path.abspath(private_dir), txtorcon)
@ -214,10 +214,10 @@ class CreateOnion(unittest.TestCase):
}
if executable:
expected["tor.executable"] = executable
self.assertEqual(tahoe_config_tor, expected)
self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1")
self.assertEqual(tor_location, "tor:ONION.onion:3457")
fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"])
self.assertEqual(dict(tor_config.node_config["tor"]), expected)
self.assertEqual(tor_config.tub_ports, ["tcp:999999:interface=127.0.0.1"])
self.assertEqual(tor_config.tub_locations, ["tor:ONION.onion:3457"])
fn = os.path.join(basedir, dict(tor_config.node_config["tor"])["onion.private_key_file"])
with open(fn, "rb") as f:
privkey = f.read()
self.assertEqual(privkey, b"privkey")
@ -251,7 +251,7 @@ class CreateOnion(unittest.TestCase):
with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
return_value=999999):
d = tor_provider.create_config(reactor, cli_config)
tahoe_config_tor, tor_port, tor_location = self.successResultOf(d)
tor_config = self.successResultOf(d)
connect_to_tor.assert_called_with(reactor, cli_config, txtorcon)
txtorcon.EphemeralHiddenService.assert_called_with("3457 127.0.0.1:999999")
@ -265,10 +265,10 @@ class CreateOnion(unittest.TestCase):
"onion.private_key_file": os.path.join("private",
"tor_onion.privkey"),
}
self.assertEqual(tahoe_config_tor, expected)
self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1")
self.assertEqual(tor_location, "tor:ONION.onion:3457")
fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"])
self.assertEqual(dict(tor_config.node_config["tor"]), expected)
self.assertEqual(tor_config.tub_ports, ["tcp:999999:interface=127.0.0.1"])
self.assertEqual(tor_config.tub_locations, ["tor:ONION.onion:3457"])
fn = os.path.join(basedir, dict(tor_config.node_config["tor"])["onion.private_key_file"])
with open(fn, "rb") as f:
privkey = f.read()
self.assertEqual(privkey, b"privkey")

17
src/allmydata/util/dictutil.py
View File

@ -2,6 +2,23 @@
Tools to mess with dicts.
"""
from __future__ import annotations
from typing import Callable, TypeVar
K = TypeVar("K")
V = TypeVar("V")
def filter(pred: Callable[[V], bool], orig: dict[K, V]) -> dict[K, V]:
"""
Filter out key/value pairs whose value fails to match a predicate.
"""
return {
k: v
for (k, v)
in orig.items()
if pred(v)
}
class DictOfSets(dict):
def add(self, key, value):
if key in self:

64
src/allmydata/util/i2p_provider.py
View File

@ -1,14 +1,9 @@
# -*- coding: utf-8 -*-
"""
Ported to Python 3.
"""
from __future__ import absolute_import, print_function, with_statement
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
from typing import Any
from typing_extensions import Literal
import os
@ -20,12 +15,15 @@ from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.endpoints import clientFromString
from twisted.internet.error import ConnectionRefusedError, ConnectError
from twisted.application import service
from twisted.python.usage import Options
from ..listeners import ListenerConfig
from ..interfaces import (
IAddressFamily,
)
from ..node import _Config
def create(reactor, config):
def create(reactor: Any, config: _Config) -> IAddressFamily:
"""
Create a new Provider service (this is an IService so must be
hooked up to a parent or otherwise started).
@ -55,6 +53,21 @@ def _import_txi2p():
except ImportError: # pragma: no cover
return None
def is_available() -> bool:
"""
Can this type of listener actually be used in this runtime
environment?
If its dependencies are missing then it cannot be.
"""
return not (_import_i2p() is None or _import_txi2p() is None)
def can_hide_ip() -> Literal[True]:
"""
Can the transport supported by this type of listener conceal the
node's public internet address from peers?
"""
return True
def _try_to_connect(reactor, endpoint_desc, stdout, txi2p):
# yields True or None
@ -97,29 +110,35 @@ def _connect_to_i2p(reactor, cli_config, txi2p):
else:
raise ValueError("unable to reach any default I2P SAM port")
@inlineCallbacks
def create_config(reactor, cli_config):
async def create_config(reactor: Any, cli_config: Options) -> ListenerConfig:
"""
For a given set of command-line options, construct an I2P listener.
This includes allocating a new I2P address.
"""
txi2p = _import_txi2p()
if not txi2p:
raise ValueError("Cannot create I2P Destination without txi2p. "
"Please 'pip install tahoe-lafs[i2p]' to fix this.")
tahoe_config_i2p = {} # written into tahoe.cfg:[i2p]
tahoe_config_i2p = [] # written into tahoe.cfg:[i2p]
private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private"))
stdout = cli_config.stdout
# XXX We shouldn't carry stdout around by jamming it into the Options
# value. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4048
stdout = cli_config.stdout # type: ignore[attr-defined]
if cli_config["i2p-launch"]:
raise NotImplementedError("--i2p-launch is under development.")
else:
print("connecting to I2P (to allocate .i2p address)..", file=stdout)
sam_port = yield _connect_to_i2p(reactor, cli_config, txi2p)
sam_port = await _connect_to_i2p(reactor, cli_config, txi2p)
print("I2P connection established", file=stdout)
tahoe_config_i2p["sam.port"] = sam_port
tahoe_config_i2p.append(("sam.port", sam_port))
external_port = 3457 # TODO: pick this randomly? there's no contention.
privkeyfile = os.path.join(private_dir, "i2p_dest.privkey")
sam_endpoint = clientFromString(reactor, sam_port)
print("allocating .i2p address...", file=stdout)
dest = yield txi2p.generateDestination(reactor, privkeyfile, 'SAM', sam_endpoint)
dest = await txi2p.generateDestination(reactor, privkeyfile, 'SAM', sam_endpoint)
print(".i2p address allocated", file=stdout)
i2p_port = "listen:i2p" # means "see [i2p]", calls Provider.get_listener()
i2p_location = "i2p:%s:%d" % (dest.host, external_port)
@ -132,10 +151,11 @@ def create_config(reactor, cli_config):
# * "private_key_file" points to the on-disk copy of the private key
# material (although we always write it to the same place)
tahoe_config_i2p["dest"] = "true"
tahoe_config_i2p["dest.port"] = str(external_port)
tahoe_config_i2p["dest.private_key_file"] = os.path.join("private",
"i2p_dest.privkey")
tahoe_config_i2p.extend([
("dest", "true"),
("dest.port", str(external_port)),
("dest.private_key_file", os.path.join("private", "i2p_dest.privkey")),
])
# tahoe_config_i2p: this is a dictionary of keys/values to add to the
# "[i2p]" section of tahoe.cfg, which tells the new node how to launch
@ -149,7 +169,7 @@ def create_config(reactor, cli_config):
# at both create-node and startup time. The data directory is not
# recorded in tahoe.cfg
returnValue((tahoe_config_i2p, i2p_port, i2p_location))
return ListenerConfig([i2p_port], [i2p_location], {"i2p": tahoe_config_i2p})
@implementer(IAddressFamily)

12
src/allmydata/util/iputil.py
View File

@ -1,17 +1,10 @@
"""
Utilities for getting IP addresses.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import native_str
from future.utils import PY2, native_str
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from typing import Callable
import os, socket
@ -39,6 +32,7 @@ from .gcutil import (
fcntl = requireModule("fcntl")
allocate_tcp_port: Callable[[], int]
from foolscap.util import allocate_tcp_port # re-exported
try:

58
src/allmydata/util/tor_provider.py
View File

@ -1,11 +1,8 @@
# -*- coding: utf-8 -*-
"""
Ported to Python 3.
"""
from __future__ import annotations
from typing import Optional
from typing import Any
from typing_extensions import Literal
import os
from zope.interface import (
@ -16,12 +13,14 @@ from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.internet.endpoints import clientFromString, TCP4ServerEndpoint
from twisted.internet.error import ConnectionRefusedError, ConnectError
from twisted.application import service
from twisted.python.usage import Options
from .observer import OneShotObserverList
from .iputil import allocate_tcp_port
from ..interfaces import (
IAddressFamily,
)
from ..listeners import ListenerConfig
def _import_tor():
@ -38,7 +37,13 @@ def _import_txtorcon():
except ImportError: # pragma: no cover
return None
def create(reactor, config, import_tor=None, import_txtorcon=None) -> Optional[_Provider]:
def can_hide_ip() -> Literal[True]:
return True
def is_available() -> bool:
return not (_import_tor() is None or _import_txtorcon() is None)
def create(reactor, config, import_tor=None, import_txtorcon=None) -> _Provider:
"""
Create a new _Provider service (this is an IService so must be
hooked up to a parent or otherwise started).
@ -150,31 +155,32 @@ def _connect_to_tor(reactor, cli_config, txtorcon):
else:
raise ValueError("unable to reach any default Tor control port")
@inlineCallbacks
def create_config(reactor, cli_config):
async def create_config(reactor: Any, cli_config: Options) -> ListenerConfig:
txtorcon = _import_txtorcon()
if not txtorcon:
raise ValueError("Cannot create onion without txtorcon. "
"Please 'pip install tahoe-lafs[tor]' to fix this.")
tahoe_config_tor = {} # written into tahoe.cfg:[tor]
tahoe_config_tor = [] # written into tahoe.cfg:[tor]
private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private"))
stdout = cli_config.stdout
# XXX We shouldn't carry stdout around by jamming it into the Options
# value. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4048
stdout = cli_config.stdout # type: ignore[attr-defined]
if cli_config["tor-launch"]:
tahoe_config_tor["launch"] = "true"
tahoe_config_tor.append(("launch", "true"))
tor_executable = cli_config["tor-executable"]
if tor_executable:
tahoe_config_tor["tor.executable"] = tor_executable
tahoe_config_tor.append(("tor.executable", tor_executable))
print("launching Tor (to allocate .onion address)..", file=stdout)
(_, tor) = yield _launch_tor(
(_, tor) = await _launch_tor(
reactor, tor_executable, private_dir, txtorcon)
tor_control_proto = tor.protocol
print("Tor launched", file=stdout)
else:
print("connecting to Tor (to allocate .onion address)..", file=stdout)
(port, tor_control_proto) = yield _connect_to_tor(
(port, tor_control_proto) = await _connect_to_tor(
reactor, cli_config, txtorcon)
print("Tor connection established", file=stdout)
tahoe_config_tor["control.port"] = port
tahoe_config_tor.append(("control.port", port))
external_port = 3457 # TODO: pick this randomly? there's no contention.
@ -183,12 +189,12 @@ def create_config(reactor, cli_config):
"%d 127.0.0.1:%d" % (external_port, local_port)
)
print("allocating .onion address (takes ~40s)..", file=stdout)
yield ehs.add_to_tor(tor_control_proto)
await ehs.add_to_tor(tor_control_proto)
print(".onion address allocated", file=stdout)
tor_port = "tcp:%d:interface=127.0.0.1" % local_port
tor_location = "tor:%s:%d" % (ehs.hostname, external_port)
privkey = ehs.private_key
yield ehs.remove_from_tor(tor_control_proto)
await ehs.remove_from_tor(tor_control_proto)
# in addition to the "how to launch/connect-to tor" keys above, we also
# record information about the onion service into tahoe.cfg.
@ -200,12 +206,12 @@ def create_config(reactor, cli_config):
# * "private_key_file" points to the on-disk copy of the private key
# material (although we always write it to the same place)
tahoe_config_tor["onion"] = "true"
tahoe_config_tor["onion.local_port"] = str(local_port)
tahoe_config_tor["onion.external_port"] = str(external_port)
assert privkey
tahoe_config_tor["onion.private_key_file"] = os.path.join("private",
"tor_onion.privkey")
tahoe_config_tor.extend([
("onion", "true"),
("onion.local_port", str(local_port)),
("onion.external_port", str(external_port)),
("onion.private_key_file", os.path.join("private", "tor_onion.privkey")),
])
privkeyfile = os.path.join(private_dir, "tor_onion.privkey")
with open(privkeyfile, "wb") as f:
if isinstance(privkey, str):
@ -224,7 +230,11 @@ def create_config(reactor, cli_config):
# at both create-node and startup time. The data directory is not
# recorded in tahoe.cfg
returnValue((tahoe_config_tor, tor_port, tor_location))
return ListenerConfig(
[tor_port],
[tor_location],
{"tor": tahoe_config_tor},
)
@implementer(IAddressFamily)