From 2db38e0cf3d4f4e788e888b17f94fb8d0b1f37e0 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 13:16:28 -0700 Subject: [PATCH 1/8] document tor/i2p/connections --- docs/configuration.rst | 230 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 221 insertions(+), 9 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 7a9189797..034f31df0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -6,15 +6,16 @@ Configuring a Tahoe-LAFS node 1. `Node Types`_ 2. `Overall Node Configuration`_ -3. `Client Configuration`_ -4. `Storage Server Configuration`_ -5. `Frontend Configuration`_ -6. `Running A Helper`_ -7. `Running An Introducer`_ -8. `Other Files in BASEDIR`_ -9. `Static Server Definitions`_ -10. `Other files`_ -11. `Example`_ +3. `Connection Management`_ +4. `Client Configuration`_ +5. `Storage Server Configuration`_ +6. `Frontend Configuration`_ +7. `Running A Helper`_ +8. `Running An Introducer`_ +9. `Other Files in BASEDIR`_ +10. `Static Server Definitions`_ +11. `Other files`_ +12. `Example`_ A Tahoe-LAFS node is configured by writing to files in its base directory. These files are read by the node when it starts, so each time you change @@ -317,6 +318,183 @@ set the ``tub.location`` option described below. string will be interpreted relative to the node's base directory. +Connection Management +===================== + +Three sections (``[tor]``, ``[i2p]``, and ``[connections]``) control how the +Tahoe node makes outbound connections. Tor and I2P are configured here. This +also controls when Tor and I2P are used: for all TCP connections (to hide +your IP address), or only when necessary (just for servers which declare that +they need Tor, because they use ``.onion`` addresses). + +``[connections]`` +----------------- + +This section controls *when* Tor and I2P are used. The ``[tor]`` and +``[i2p]`` sections (described later) control *how* Tor/I2P connections are +managed. + +All Tahoe nodes need to make a connection to the Introducer; the ``[node] +introducer.furl`` setting (described below) indicates where the Introducer +lives. Tahoe client nodes must also make connections to storage servers: +these targets are specified in announcements that come from the Introducer. +Both are expressed as FURLs (a Foolscap URL), which include a list of +"connection hints". Each connection hint describes one (of perhaps many) +network endpoints where the service might live. + +Connection hints include a type, and look like: + +* ``tcp:tahoe.example.org:12345`` +* ``tor:u33m4y7klhz3b.onion:1000`` +* ``i2p:c2ng2pbrmxmlwpijn`` + +``tor`` hints are always handled by the ``tor`` handler (configured in the +``[tor]`` section, described below). Likewise, ``i2p`` hints are always +routed to the ``i2p`` handler. But either will be ignored if Tahoe was not +installed with the necessary Tor/I2P support libraries, or if the Tor/I2P +daemon is unreachable. + +The ``[connections]`` section lets you control how ``tcp`` hints are handled. +By default, they use the normal TCP handler, which just makes direct +connections (revealing your node's IP address to both the target server and +the intermediate network). The node behaves this way if the ``[connections]`` +section is missing entirely, or if it looks like this:: + + [connections] + tcp = tcp + +To hide the Tahoe node's IP address from the servers that it uses, set the +``[connections]`` section to use Tor for TCP hints:: + + [connections] + tcp = tor + +(Note that I2P does not support connections to normal TCP ports, so +``[connections] tcp = i2p`` is invalid) + +In the future, Tahoe services may be changed to live on HTTP/HTTPS URLs +instead of Foolscap. In that case, connections will be made using whatever +handler is configured for ``tcp`` hints. So the same ``tcp = tor`` +configuration will work. + +``[tor]`` +--------- + +This controls how Tor connections are made. The defaults (all empty) mean +that, when Tor is needed, the node will try to connect to a Tor daemon's +SOCKS proxy on localhost port 9050 or 9150. Port 9050 is the default Tor +SOCKS port, so it should be available under any system Tor instance (e.g. the +one launched at boot time when the standard Debian ``tor`` package is +installed). Port 9150 is the SOCKS port for the Tor Browser Bundle, so it +will be available any time the TBB is running. + +You can set ``launch = True`` to cause the Tahoe node to launch a new Tor +daemon when it starts up (and kill it at shutdown), if you don't have a +system-wide instance available. Note that it takes 30-60 seconds for Tor to +get running, so using a long-running Tor process may enable a faster startup. +If your Tor executable doesn't live on ``$PATH``, use ``tor.executable=`` to +specify it. + +``[tor]`` + +``enable = (boolean, optional, defaults to True)`` + + If False, this will disable the use of Tor entirely. The default of True + means the node will use Tor, if necessary, and if possible. + +``socks.port = (string, optional, PORT, defaults to empty)`` + + This tells the node that Tor connections should be routed to a SOCKS + proxy listening on the given port. The default (of an empty value) will + cause the node to first try localhost port 9050, then if that fails, try + localhost port 9150. These are the default listening ports of the + standard Tor daemon, and the Tor Browser Bundle, respectively. + +``control.port = (string, optional, endpoint specification string)`` + + This tells the node to connect to a pre-existing Tor daemon on the given + control port (which is typically ``unix://var/run/tor/control`` or + ``tcp:localhost:9051``). The node will then ask Tor what SOCKS port it is + using, and route Tor connections to that. + +``launch = (bool, optional, defaults to False)`` + + If True, the node will spawn a new (private) copy of Tor at startup, and + will kill it at shutdown. The new Tor will be given a persistent state + directory under ``NODEDIR/private/``, where Tor's microdescriptors will + be cached, to speed up subsequent startup. + +``tor.executable = (string, optional, defaults to empty)`` + + This controls which Tor executable is used when ``launch = True``. If + empty, the first executable program named ``tor`` found on ``$PATH`` will + be used. + +There are 5 valid combinations of these configuration settings: + +* 1: ``(empty)``: use SOCKS on port 9050/9150 +* 2: ``launch = true``: launch a new Tor +* 3: ``socks.port = HOST:PORT``: use an existing Tor on the given SOCKS port +* 4: ``control.port = PORT``: use an existing Tor at the given control port +* 5: ``enable = false``: no Tor at all + +1 is the default, and should work for any Linux host with the system Tor +package installed. 2 should work on any box with Tor installed into $PATH, +but will take an extra 30-60 seconds at startup. 3 and 4 can be used for +specialized installations, where Tor is already running, but not listening on +the default port. 5 should be used in environments where Tor is installed, +but should not be used (perhaps due to a site-wide policy). + +Note that Tor support depends upon some additional Python libraries. To +install Tahoe with Tor support, use ``pip install tahoe-lafs[tor]``. + +``[i2p]`` +--------- + +This controls how I2P connections are made. Like with Tor, the all-empty +defaults will cause I2P connections to be routed to a pre-existing I2P daemon +on port 7656. This is the default SAM port for the ``i2p`` daemon. + + +``[i2p]`` + +``enable = (boolean, optional, defaults to True)`` + + If False, this will disable the use of I2P entirely. The default of True + means the node will use I2P, if necessary, and if possible. + +``sam.port = (string, optional, endpoint descriptor, defaults to empty)`` + + This tells the node that I2P connections should be made via the SAM + protocol on the given port. The default (of an empty value) will cause + the node to try localhost port 7656. This is the default listening port + of the standard I2P daemon. + +``launch = (bool, optional, defaults to False)`` + + If True, the node will spawn a new (private) copy of I2P at startup, and + will kill it at shutdown. The new I2P will be given a persistent state + directory under ``NODEDIR/private/``, where I2P's microdescriptors will + be cached, to speed up subsequent startup. The daemon will allocate its + own SAM port, which will be queried from the config directory. + +``i2p.configdir = (string, optional, directory)`` + + This tells the node to parse an I2P config file in the given directory, + and use the SAM port it finds there. If ``launch = True``, the new I2P + daemon will be told to use the given directory (which can be + pre-populated with a suitable config file). If ``launch = False``, we + assume there is a pre-running I2P daemon running from this directory, and + can again parse the config file for the SAM port. + +``i2p.executable = (string, optional, defaults to empty)`` + + This controls which I2P executable is used when ``launch = True``. If + empty, the first executable program named ``i2p`` found on ``$PATH`` will + be used. + + + Client Configuration ==================== @@ -569,6 +747,8 @@ Other Files in BASEDIR Some configuration is not kept in ``tahoe.cfg``, for the following reasons: +* it doesn't fit into the INI format of ``tahoe.cfg`` (e.g. + ``private/servers.yaml``) * it is generated by the node at startup, e.g. encryption keys. The node never writes to ``tahoe.cfg``. * it is generated by user action, e.g. the "``tahoe create-alias``" command. @@ -740,6 +920,38 @@ Or, if you're feeling really lazy:: .. _YAML: http://yaml.org/ +Overriding Connection-Handlers for Static Servers +------------------------------------------------- + +A ``connections`` entry will override the default connection-handler mapping +(as established by ``tahoe.cfg [connections]``). This can be used to build a +"Tor-mostly client": one which is restricted to use Tor for all connections, +except for a few private servers to which normal TCP connections will be +made. To override the published announcement (and thus avoid connecting twice +to the same server), the server ID must exactly match. + +``tahoe.cfg``:: + + [connections] + # this forces the use of Tor for all "tcp" hints + tcp = tor + +``private/servers.yaml``:: + + storage: + v0-c2ng2pbrmxmlwpijn3mr72ckk5fmzk6uxf6nhowyosaubrt6y5mq: + ann: + nickname: my-server-1 + anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:10.1.2.3:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b + connections: + # this overrides the tcp=tor from tahoe.cfg, for just this server + tcp: tcp + +The ``connections`` table is needed to override the ``tcp = tor`` mapping +that comes from ``tahoe.cfg``. Without it, the client would attempt to use +Tor to connect to ``10.1.2.3``, which would fail because it is a +local/non-routeable (RFC1918) address. + Other files =========== From 73d5376b6adf3530bfc37b399ab4fefb744f57a4 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 16:54:01 -0700 Subject: [PATCH 2/8] Implement basic connection handlers (tor/i2p) The node now attempts to create Tor/I2P connection handlers (if the right libraries are available), and will use them for tor/i2p FURL hints by default. For now it only creates default handlers: there is not yet any code to interpret the `[tor]`/`[i2p]` sections of tahoe.cfg which would let you override that process. The node also parses the `[connections]` section, allowing `tcp: tor` to use Tor for all outbound TCP connections. It defaults to `tcp: tcp`, of course. Static storage-server connections will now honor the `connections:` overrides in `servers.yaml`, allowing specific servers to use TCP where they would normally be restricted to Tor. refs ticket:2788 refs ticket:517 --- src/allmydata/node.py | 47 +++++++++++++++++++++++++++++++- src/allmydata/test/no_network.py | 2 ++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index c35ffcc9d..140d2d25f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -83,6 +83,7 @@ class Node(service.MultiService): assert type(self.nickname) is unicode self.init_tempdir() + self.init_connections() self.set_tub_options() self.create_main_tub() self.create_control_tub() @@ -164,6 +165,44 @@ class Node(service.MultiService): twlog.msg(e) raise e + def _make_tcp_handler(self): + # this is always available + from foolscap.connections.tcp import default + return default() + + def _make_tor_handler(self): + try: + # TODO: parse [tor] config, build handler to match + from foolscap.connections.tor import default_socks + return default_socks() + except ImportError: + return None + + def _make_i2p_handler(self): + # TODO: parse [i2p] config, build handler to match + return None + + def init_connections(self): + # We store handlers for everything. None means we were unable to + # create that handler, so hints which want it will be ignored. + handlers = self._foolscap_connection_handlers = { + "tcp": self._make_tcp_handler(), + "tor": self._make_tor_handler(), + "i2p": self._make_i2p_handler(), + } + self.log("built Foolscap connection handlers for: %(known_handlers)s", + known_handlers=sorted([k for k,v in handlers.items() if v]), + facility="tahoe.node", umid="PuLh8g") + + # then we remember the default mappings from tahoe.cfg + self._default_connection_handlers = {"tor": "tor", "i2p": "i2p"} + tcp_handler_name = self.get_config("connections", "tcp", "tcp").lower() + if tcp_handler_name not in handlers: + raise ValueError("'tahoe.cfg [connections] tcp='" + " uses unknown handler type '%s'" + % tcp_handler_name) + self._default_connection_handlers["tcp"] = tcp_handler_name + def set_tub_options(self): self.tub_options = { "logLocalFailures": True, @@ -181,12 +220,18 @@ class Node(service.MultiService): self.tub_options["disconnectTimeout"] = int(disconnect_timeout_s) def _create_tub(self, handler_overrides={}, **kwargs): - assert not handler_overrides # Create a Tub with the right options and handlers. It will be # ephemeral unless the caller provides certFile= tub = Tub(**kwargs) for (name, value) in self.tub_options.items(): tub.setOption(name, value) + handlers = self._default_connection_handlers.copy() + handlers.update(handler_overrides) + tub.removeAllConnectionHintHandlers() + for hint_type, handler_name in handlers.items(): + handler = self._foolscap_connection_handlers.get(handler_name) + if handler: + tub.addConnectionHintHandler(hint_type, handler) return tub def _convert_tub_port(self, s): diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index d728b6819..df3c88137 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -180,6 +180,8 @@ class NoNetworkStorageBroker: class NoNetworkClient(Client): + def init_connections(self): + pass def create_main_tub(self): pass def init_introducer_client(self): From 3cc8daf5bd0b5774103d396f536a54d0528007fa Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 18:23:37 -0700 Subject: [PATCH 3/8] implement full range of Tor handlers --- src/allmydata/node.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 140d2d25f..ec9fe600a 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -1,6 +1,7 @@ import datetime, os.path, re, types, ConfigParser, tempfile from base64 import b32decode, b32encode +from twisted.internet import reactor, endpoints from twisted.python import log as twlog from twisted.application import service from foolscap.api import Tub, app_versions @@ -171,13 +172,48 @@ class Node(service.MultiService): return default() def _make_tor_handler(self): + enabled = self.get_config("tor", "enable", True, boolean=True) + if not enabled: + return None try: - # TODO: parse [tor] config, build handler to match - from foolscap.connections.tor import default_socks - return default_socks() + from foolscap.connections import tor except ImportError: return None + if self.get_config("tor", "launch", False, boolean=True): + executable = self.get_config("tor", "tor.executable", None) + datadir = os.path.join(self.basedir, "private", "tor-statedir") + return tor.launch(data_directory=datadir, tor_binary=executable) + + socksport = self.get_config("tor", "socks.port", None) + if socksport: + # foolscap.connections.tor.socks_port() in Foolscap-0.12.1 only + # allows the use of SOCKS port on localhost, to discourage unsafe + # connections to remote SOCKS ports. Allow the HOST:PORT syntax, + # but refuse to use anything other than 127.0.0.1 . Also accept + # just PORT. + if ":" in socksport: + host, port = socksport.split(":") + if host != "127.0.0.1": + raise ValueError("'tahoe.cfg [tor] socks.port' = " + "must be '127.0.0.1:PORT' or just PORT, " + "not '%s'" % (socksport,)) + else: + port = socksport + try: + port = int(port) + except ValueError: + raise ValueError("'tahoe.cfg [tor] socks.port' used " + "non-numeric PORT value '%s'" % (port,)) + return tor.socks_port(port) + + controlport = self.get_config("tor", "control.port", None) + if controlport: + ep = endpoints.clientFromString(reactor, controlport) + return tor.control_endpoint(ep) + + return tor.default_socks() + def _make_i2p_handler(self): # TODO: parse [i2p] config, build handler to match return None From c56a0a4ba9a93d636e0840e53bcd3e94337edf1b Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 19:41:20 -0700 Subject: [PATCH 4/8] fix unicode handling in server_id from YAML YAML, like JSON, is all-unicode. StorageFarmBroker.set_static_servers() is defined to take an all-unicode dictionary (the "storage:" key from private/servers.yaml), so the server_id keys it gets will be unicode. NativeStorageServer is defined to accept server_ids which are bytes (at least it is now). The tests were only passing bytes into set_static_servers(), whereas a real launch passed unicode in, causing a problem when NativeStorageServer tried to base32.a2b() the pubkey and choked on the unicode it received. This fixes set_static_servers() to convert the server_id to bytes, and changes NativeStorageServer to assert that it gets bytes. It also fixes the test to match real usage more closely. --- src/allmydata/storage_client.py | 3 +++ src/allmydata/test/test_storage_client.py | 28 +++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 78c5b6360..ad6232390 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -86,6 +86,8 @@ class StorageFarmBroker(service.MultiService): def set_static_servers(self, servers): for (server_id, server) in servers.items(): + assert isinstance(server_id, unicode) # from YAML + server_id = server_id.encode("ascii") self._static_server_ids.add(server_id) handler_overrides = server.get("connections", {}) s = NativeStorageServer(server_id, server["ann"], @@ -275,6 +277,7 @@ class NativeStorageServer(service.MultiService): def __init__(self, server_id, ann, tub_maker, handler_overrides): service.MultiService.__init__(self) + assert isinstance(server_id, str) self._server_id = server_id self.announcement = ann self._tub_maker = tub_maker diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 592689ef9..a78f91acb 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -1,6 +1,6 @@ import hashlib from mock import Mock -from allmydata.util import base32 +from allmydata.util import base32, yamlutil from twisted.trial import unittest from twisted.internet.defer import succeed, inlineCallbacks @@ -48,17 +48,21 @@ class TestStorageFarmBroker(unittest.TestCase): def test_static_servers(self): broker = StorageFarmBroker(True, lambda h: Mock()) - key_s = 'v0-1234-{}'.format(1) - ann = { - "service-name": "storage", - "anonymous-storage-FURL": "pb://{}@nowhere/fake".format(base32.b2a(str(1))), - "permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa", - } + key_s = 'v0-1234-1' + servers_yaml = """\ +storage: + v0-1234-1: + ann: + anonymous-storage-FURL: pb://ge@nowhere/fake + permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa +""" + servers = yamlutil.safe_load(servers_yaml) permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa") - broker.set_static_servers({key_s: {"ann": ann}}) + broker.set_static_servers(servers["storage"]) self.failUnlessEqual(len(broker._static_server_ids), 1) s = broker.servers[key_s] - self.failUnlessEqual(s.announcement, ann) + self.failUnlessEqual(s.announcement, + servers["storage"]["v0-1234-1"]["ann"]) self.failUnlessEqual(s.get_serverid(), key_s) self.assertEqual(s.get_permutation_seed(), permseed) @@ -82,7 +86,7 @@ class TestStorageFarmBroker(unittest.TestCase): ann = { "anonymous-storage-FURL": "pb://abcde@nowhere/fake", } - broker.set_static_servers({server_id: {"ann": ann}}) + broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) s = broker.servers[server_id] self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) @@ -94,7 +98,7 @@ class TestStorageFarmBroker(unittest.TestCase): "anonymous-storage-FURL": "pb://abcde@nowhere/fake", "permutation-seed-base32": k, } - broker.set_static_servers({server_id: {"ann": ann}}) + broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) s = broker.servers[server_id] self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) @@ -104,7 +108,7 @@ class TestStorageFarmBroker(unittest.TestCase): ann = { "anonymous-storage-FURL": "pb://abcde@nowhere/fake", } - broker.set_static_servers({server_id: {"ann": ann}}) + broker.set_static_servers({server_id.decode("ascii"): {"ann": ann}}) s = broker.servers[server_id] self.assertEqual(s.get_permutation_seed(), hashlib.sha256(server_id).digest()) From eb230d1e5fc65bfc6eb7e367b17600bdf20cb1a5 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 19:57:01 -0700 Subject: [PATCH 5/8] setup.py: add 'tox' and 'i2p' extras Now you can do `pip install tahoe-lafs[tor]` to get tor support. The necessary support libraries are also installed with `[test]`, so unit tests can rely upon importing txtorcon and friends. --- setup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup.py b/setup.py index 50b872ca2..68a7c9641 100644 --- a/setup.py +++ b/setup.py @@ -261,6 +261,18 @@ setup(name="tahoe-lafs", # also set in __init__.py "coverage", "mock", "tox", + "foolscap[tor]", + "txtorcon", # in case pip's resolver doesn't work + "foolscap[i2p]", + "txi2p", # in case pip's resolver doesn't work + ], + "tor": [ + "foolscap[tor]", + "txtorcon", # in case pip's resolver doesn't work + ], + "i2p": [ + "foolscap[i2p]", + "txi2p", # in case pip's resolver doesn't work ], }, package_data={"allmydata.web": ["*.xhtml", From 15e5ca0e996c0a6f546c0ed34fcdd4fa5e8164d5 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 28 Aug 2016 02:28:39 -0700 Subject: [PATCH 6/8] exercise TCP/Tor-related tahoe.cfg parsing This exercises everything about _make_tcp_handler() and _make_tor_handler() except for when txtorcon cannot be imported. --- src/allmydata/test/test_connections.py | 125 +++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 src/allmydata/test/test_connections.py diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py new file mode 100644 index 000000000..3e9c14dae --- /dev/null +++ b/src/allmydata/test/test_connections.py @@ -0,0 +1,125 @@ +import os +import mock +from io import BytesIO +from twisted.trial import unittest +from twisted.internet import reactor, endpoints +from ConfigParser import SafeConfigParser +from foolscap.connections import tcp +from ..node import Node + +class FakeNode(Node): + def __init__(self, config_str): + self.config = SafeConfigParser() + self.config.readfp(BytesIO(config_str)) + +BASECONFIG = ("[client]\n" + "introducer.furl = \n" + ) + + +class TCP(unittest.TestCase): + def test_default(self): + n = FakeNode(BASECONFIG) + h = n._make_tcp_handler() + self.assertIsInstance(h, tcp.DefaultTCP) + +class Tor(unittest.TestCase): + def test_disabled(self): + n = FakeNode(BASECONFIG+"[tor]\nenable = false\n") + h = n._make_tor_handler() + self.assertEqual(h, None) + + def test_default(self): + n = FakeNode(BASECONFIG) + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.default_socks", + return_value=h1) as f: + h = n._make_tor_handler() + self.assertEqual(f.mock_calls, [mock.call()]) + self.assertIdentical(h, h1) + + def test_launch(self): + n = FakeNode(BASECONFIG+"[tor]\nlaunch = true\n") + n.basedir = "BASEDIR" + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.launch", + return_value=h1) as f: + h = n._make_tor_handler() + data_directory = os.path.join(n.basedir, "private", "tor-statedir") + exp = mock.call(data_directory=data_directory, + tor_binary=None) + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_launch_executable(self): + n = FakeNode(BASECONFIG+"[tor]\nlaunch = true\ntor.executable = tor") + n.basedir = "BASEDIR" + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.launch", + return_value=h1) as f: + h = n._make_tor_handler() + data_directory = os.path.join(n.basedir, "private", "tor-statedir") + exp = mock.call(data_directory=data_directory, + tor_binary="tor") + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_socksport(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = 1234\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.socks_port", + return_value=h1) as f: + h = n._make_tor_handler() + self.assertEqual(f.mock_calls, [mock.call(1234)]) + self.assertIdentical(h, h1) + + def test_socksport_localhost(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = 127.0.0.1:1234\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.socks_port", + return_value=h1) as f: + h = n._make_tor_handler() + self.assertEqual(f.mock_calls, [mock.call(1234)]) + self.assertIdentical(h, h1) + + def test_socksport_bad_host(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = example.com:1234\n") + e = self.assertRaises(ValueError, n._make_tor_handler) + self.assertIn("must be '127.0.0.1:PORT'", str(e)) + + def test_socksport_not_integer(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = kumquat\n") + e = self.assertRaises(ValueError, n._make_tor_handler) + self.assertIn("used non-numeric PORT value", str(e)) + + def test_controlport(self): + n = FakeNode(BASECONFIG+"[tor]\ncontrol.port = tcp:localhost:1234\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.tor.control_endpoint", + return_value=h1) as f: + h = n._make_tor_handler() + self.assertEqual(len(f.mock_calls), 1) + ep = f.mock_calls[0][1][0] + self.assertIsInstance(ep, endpoints.TCP4ClientEndpoint) + self.assertIdentical(h, h1) + +class Connections(unittest.TestCase): + def test_default(self): + n = FakeNode(BASECONFIG) + n.init_connections() + self.assertEqual(n._default_connection_handlers["tcp"], "tcp") + self.assertEqual(n._default_connection_handlers["tor"], "tor") + self.assertEqual(n._default_connection_handlers["i2p"], "i2p") + + def test_tor(self): + n = FakeNode(BASECONFIG+"[connections]\ntcp = tor\n") + n.init_connections() + self.assertEqual(n._default_connection_handlers["tcp"], "tor") + self.assertEqual(n._default_connection_handlers["tor"], "tor") + self.assertEqual(n._default_connection_handlers["i2p"], "i2p") + + def test_unknown(self): + n = FakeNode(BASECONFIG+"[connections]\ntcp = unknown\n") + e = self.assertRaises(ValueError, n.init_connections) + self.assertIn("'tahoe.cfg [connections] tcp='", str(e)) + self.assertIn("uses unknown handler type 'unknown'", str(e)) From bc079a71ebbb700db5ad2390a71db5d6170ccd09 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 28 Aug 2016 03:15:35 -0700 Subject: [PATCH 7/8] implement+test I2P setup Note that many of the Foolscap handler-creation functions are still stubbed out, so Tahoe won't be able to honor the full range of config syntax until foolscap support is complete. --- src/allmydata/node.py | 29 ++++++++- src/allmydata/test/test_connections.py | 86 ++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index ec9fe600a..8a1168c4d 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -215,8 +215,33 @@ class Node(service.MultiService): return tor.default_socks() def _make_i2p_handler(self): - # TODO: parse [i2p] config, build handler to match - return None + enabled = self.get_config("i2p", "enable", True, boolean=True) + if not enabled: + return None + try: + from foolscap.connections import i2p + except ImportError: + return None + + samport = self.get_config("i2p", "sam.port", None) + launch = self.get_config("i2p", "launch", False, boolean=True) + configdir = self.get_config("i2p", "i2p.configdir", None) + + if samport: + if launch: + raise ValueError("tahoe.cfg [i2p] must not set both " + "sam.port and launch") + ep = endpoints.clientFromString(reactor, samport) + return i2p.sam_endpoint(ep) + + if launch: + executable = self.get_config("i2p", "i2p.executable", None) + return i2p.launch(i2p_configdir=configdir, i2p_binary=executable) + + if configdir: + return i2p.local_i2p(configdir) + + return i2p.default(reactor) def init_connections(self): # We store handlers for everything. None means we were unable to diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 3e9c14dae..9f3ad2113 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -103,6 +103,92 @@ class Tor(unittest.TestCase): self.assertIsInstance(ep, endpoints.TCP4ClientEndpoint) self.assertIdentical(h, h1) +class I2P(unittest.TestCase): + def test_disabled(self): + n = FakeNode(BASECONFIG+"[i2p]\nenable = false\n") + h = n._make_i2p_handler() + self.assertEqual(h, None) + + def test_default(self): + n = FakeNode(BASECONFIG) + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.default", + return_value=h1) as f: + h = n._make_i2p_handler() + self.assertEqual(f.mock_calls, [mock.call(reactor)]) + self.assertIdentical(h, h1) + + def test_samport(self): + n = FakeNode(BASECONFIG+"[i2p]\nsam.port = tcp:localhost:1234\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.sam_endpoint", + return_value=h1) as f: + h = n._make_i2p_handler() + self.assertEqual(len(f.mock_calls), 1) + ep = f.mock_calls[0][1][0] + self.assertIsInstance(ep, endpoints.TCP4ClientEndpoint) + self.assertIdentical(h, h1) + + def test_samport_and_launch(self): + n = FakeNode(BASECONFIG+"[i2p]\n" + + "sam.port = tcp:localhost:1234\n" + +"launch = true\n") + e = self.assertRaises(ValueError, n._make_i2p_handler) + self.assertIn("must not set both sam.port and launch", str(e)) + + def test_launch(self): + n = FakeNode(BASECONFIG+"[i2p]\nlaunch = true\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.launch", + return_value=h1) as f: + h = n._make_i2p_handler() + exp = mock.call(i2p_configdir=None, i2p_binary=None) + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_launch_executable(self): + n = FakeNode(BASECONFIG+"[i2p]\nlaunch = true\n" + + "i2p.executable = i2p\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.launch", + return_value=h1) as f: + h = n._make_i2p_handler() + exp = mock.call(i2p_configdir=None, i2p_binary="i2p") + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_launch_configdir(self): + n = FakeNode(BASECONFIG+"[i2p]\nlaunch = true\n" + + "i2p.configdir = cfg\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.launch", + return_value=h1) as f: + h = n._make_i2p_handler() + exp = mock.call(i2p_configdir="cfg", i2p_binary=None) + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_launch_configdir_and_executable(self): + n = FakeNode(BASECONFIG+"[i2p]\nlaunch = true\n" + + "i2p.executable = i2p\n" + + "i2p.configdir = cfg\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.launch", + return_value=h1) as f: + h = n._make_i2p_handler() + exp = mock.call(i2p_configdir="cfg", i2p_binary="i2p") + self.assertEqual(f.mock_calls, [exp]) + self.assertIdentical(h, h1) + + def test_configdir(self): + n = FakeNode(BASECONFIG+"[i2p]\ni2p.configdir = cfg\n") + h1 = mock.Mock() + with mock.patch("foolscap.connections.i2p.local_i2p", + return_value=h1) as f: + h = n._make_i2p_handler() + self.assertEqual(f.mock_calls, [mock.call("cfg")]) + self.assertIdentical(h, h1) + class Connections(unittest.TestCase): def test_default(self): n = FakeNode(BASECONFIG) From a099b9237df462c40c20c48a6240bd1f20573d5e Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sun, 28 Aug 2016 16:28:01 -0700 Subject: [PATCH 8/8] tor: socks.port is now a (restricted) endpoint string Foolscap has limitations that prevent us from accepting anything but a TCP endpoint, but that will change in the future, so make the tahoe.cfg syntax accept an endpoint, but then reject non-TCP ones. See the ticket for details: refs ticket:2813. This depends upon the new `foolscap.connections.tor.socks_port(host, port)` API in foolscap-0.12.2, so it bumps the dependency to that (the previous commit depended upon 0.12.1, but I hadn't gotten around to updating the dep before now). --- docs/configuration.rst | 19 +++++++++++------ src/allmydata/_auto_deps.py | 3 ++- src/allmydata/node.py | 29 ++++++++++++-------------- src/allmydata/test/test_connections.py | 18 ++++++++-------- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 034f31df0..f9528d1c9 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -402,14 +402,21 @@ specify it. If False, this will disable the use of Tor entirely. The default of True means the node will use Tor, if necessary, and if possible. -``socks.port = (string, optional, PORT, defaults to empty)`` +``socks.port = (string, optional, endpoint specification string, defaults to empty)`` This tells the node that Tor connections should be routed to a SOCKS - proxy listening on the given port. The default (of an empty value) will - cause the node to first try localhost port 9050, then if that fails, try - localhost port 9150. These are the default listening ports of the + proxy listening on the given endpoint. The default (of an empty value) + will cause the node to first try localhost port 9050, then if that fails, + try localhost port 9150. These are the default listening ports of the standard Tor daemon, and the Tor Browser Bundle, respectively. + While this nominally accepts an arbitrary endpoint string, internal + limitations prevent it from accepting anything but ``tcp:HOST:PORT`` + (unfortunately, unix-domain sockets are not yet supported). See ticket + #2813 for details. Also note that using a HOST of anything other than + localhost is discouraged, because you would be revealing your IP address + to external (and possibly hostile) machines. + ``control.port = (string, optional, endpoint specification string)`` This tells the node to connect to a pre-existing Tor daemon on the given @@ -434,8 +441,8 @@ There are 5 valid combinations of these configuration settings: * 1: ``(empty)``: use SOCKS on port 9050/9150 * 2: ``launch = true``: launch a new Tor -* 3: ``socks.port = HOST:PORT``: use an existing Tor on the given SOCKS port -* 4: ``control.port = PORT``: use an existing Tor at the given control port +* 3: ``socks.port = tcp:HOST:PORT``: use an existing Tor on the given SOCKS port +* 4: ``control.port = ENDPOINT``: use an existing Tor at the given control port * 5: ``enable = false``: no Tor at all 1 is the default, and should work for any Linux host with the system Tor diff --git a/src/allmydata/_auto_deps.py b/src/allmydata/_auto_deps.py index 55018fe49..831dd69c8 100644 --- a/src/allmydata/_auto_deps.py +++ b/src/allmydata/_auto_deps.py @@ -39,7 +39,8 @@ install_requires = [ # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, # rather than 1024-bit RSA-with-MD5. This also allows us to work # with a FIPS build of OpenSSL. - "foolscap >= 0.10.1", + # * foolscap >= 0.12.2 provides tcp/tor/i2p connection handlers we need + "foolscap >= 0.12.2", # Needed for SFTP. # pycrypto 2.2 doesn't work due to diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 8a1168c4d..34c1a2678 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -187,25 +187,22 @@ class Node(service.MultiService): socksport = self.get_config("tor", "socks.port", None) if socksport: - # foolscap.connections.tor.socks_port() in Foolscap-0.12.1 only - # allows the use of SOCKS port on localhost, to discourage unsafe - # connections to remote SOCKS ports. Allow the HOST:PORT syntax, - # but refuse to use anything other than 127.0.0.1 . Also accept - # just PORT. - if ":" in socksport: - host, port = socksport.split(":") - if host != "127.0.0.1": - raise ValueError("'tahoe.cfg [tor] socks.port' = " - "must be '127.0.0.1:PORT' or just PORT, " - "not '%s'" % (socksport,)) - else: - port = socksport + # this is nominally and endpoint string, but txtorcon requires + # TCP host and port. So parse it now, and reject non-TCP + # endpoints. + + pieces = socksport.split(":") + if pieces[0] != "tcp" or len(pieces) != 3: + raise ValueError("'tahoe.cfg [tor] socks.port' = " + "is currently limited to 'tcp:HOST:PORT', " + "not '%s'" % (socksport,)) + host = pieces[1] try: - port = int(port) + port = int(pieces[2]) except ValueError: raise ValueError("'tahoe.cfg [tor] socks.port' used " - "non-numeric PORT value '%s'" % (port,)) - return tor.socks_port(port) + "non-numeric PORT value '%s'" % (pieces[2],)) + return tor.socks_port(host, port) controlport = self.get_config("tor", "control.port", None) if controlport: diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 9f3ad2113..a20b15888 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -65,30 +65,30 @@ class Tor(unittest.TestCase): self.assertIdentical(h, h1) def test_socksport(self): - n = FakeNode(BASECONFIG+"[tor]\nsocks.port = 1234\n") + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = tcp:127.0.0.1:1234\n") h1 = mock.Mock() with mock.patch("foolscap.connections.tor.socks_port", return_value=h1) as f: h = n._make_tor_handler() - self.assertEqual(f.mock_calls, [mock.call(1234)]) + self.assertEqual(f.mock_calls, [mock.call("127.0.0.1", 1234)]) self.assertIdentical(h, h1) - def test_socksport_localhost(self): - n = FakeNode(BASECONFIG+"[tor]\nsocks.port = 127.0.0.1:1234\n") + def test_socksport_otherhost(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = tcp:otherhost:1234\n") h1 = mock.Mock() with mock.patch("foolscap.connections.tor.socks_port", return_value=h1) as f: h = n._make_tor_handler() - self.assertEqual(f.mock_calls, [mock.call(1234)]) + self.assertEqual(f.mock_calls, [mock.call("otherhost", 1234)]) self.assertIdentical(h, h1) - def test_socksport_bad_host(self): - n = FakeNode(BASECONFIG+"[tor]\nsocks.port = example.com:1234\n") + def test_socksport_bad_endpoint(self): + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = unix:unsupported\n") e = self.assertRaises(ValueError, n._make_tor_handler) - self.assertIn("must be '127.0.0.1:PORT'", str(e)) + self.assertIn("is currently limited to 'tcp:HOST:PORT'", str(e)) def test_socksport_not_integer(self): - n = FakeNode(BASECONFIG+"[tor]\nsocks.port = kumquat\n") + n = FakeNode(BASECONFIG+"[tor]\nsocks.port = tcp:localhost:kumquat\n") e = self.assertRaises(ValueError, n._make_tor_handler) self.assertIn("used non-numeric PORT value", str(e))