rewrite tor_provider

This uses a unix-domain control port, and includes test coverage.

create_onion() displays pacifier messages, since the allocate-onion step
takes around 35 seconds

src/allmydata/test/test_tor_provider.py

@@ -0,0 +1,532 @@
+import os
+from twisted.trial import unittest
+from twisted.internet import defer, error
+from StringIO import StringIO
+import mock
+from ..util import tor_provider
+from ..scripts import create_node, runner
+from foolscap.eventual import flushEventualQueue
+
+def mock_txtorcon(txtorcon):
+    return mock.patch("allmydata.util.tor_provider._import_txtorcon",
+                      return_value=txtorcon)
+
+def mock_tor(tor):
+    return mock.patch("allmydata.util.tor_provider._import_tor",
+                      return_value=tor)
+
+def make_cli_config(basedir, *argv):
+    parent = runner.Options()
+    cli_config = create_node.CreateNodeOptions()
+    cli_config.parent = parent
+    cli_config.parseOptions(argv)
+    cli_config["basedir"] = basedir
+    cli_config.stdout = StringIO()
+    return cli_config
+
+class TryToConnect(unittest.TestCase):
+    def test_try(self):
+        reactor = object()
+        txtorcon = mock.Mock()
+        tor_state = object()
+        d = defer.succeed(tor_state)
+        txtorcon.build_tor_connection = mock.Mock(return_value=d)
+        ep = object()
+        stdout = StringIO()
+        with mock.patch("allmydata.util.tor_provider.clientFromString",
+                        return_value=ep) as cfs:
+            d = tor_provider._try_to_connect(reactor, "desc", stdout, txtorcon)
+        r = self.successResultOf(d)
+        self.assertIs(r, tor_state)
+        cfs.assert_called_with(reactor, "desc")
+        txtorcon.build_tor_connection.assert_called_with(ep)
+
+    def test_try_handled_error(self):
+        reactor = object()
+        txtorcon = mock.Mock()
+        d = defer.fail(error.ConnectError("oops"))
+        txtorcon.build_tor_connection = mock.Mock(return_value=d)
+        ep = object()
+        stdout = StringIO()
+        with mock.patch("allmydata.util.tor_provider.clientFromString",
+                        return_value=ep) as cfs:
+            d = tor_provider._try_to_connect(reactor, "desc", stdout, txtorcon)
+        r = self.successResultOf(d)
+        self.assertIs(r, None)
+        cfs.assert_called_with(reactor, "desc")
+        txtorcon.build_tor_connection.assert_called_with(ep)
+        self.assertEqual(stdout.getvalue(),
+                         "Unable to reach Tor at 'desc': "
+                         "An error occurred while connecting: oops.\n")
+
+    def test_try_unhandled_error(self):
+        reactor = object()
+        txtorcon = mock.Mock()
+        d = defer.fail(ValueError("oops"))
+        txtorcon.build_tor_connection = mock.Mock(return_value=d)
+        ep = object()
+        stdout = StringIO()
+        with mock.patch("allmydata.util.tor_provider.clientFromString",
+                        return_value=ep) as cfs:
+            d = tor_provider._try_to_connect(reactor, "desc", stdout, txtorcon)
+        f = self.failureResultOf(d)
+        self.assertIsInstance(f.value, ValueError)
+        self.assertEqual(str(f.value), "oops")
+        cfs.assert_called_with(reactor, "desc")
+        txtorcon.build_tor_connection.assert_called_with(ep)
+        self.assertEqual(stdout.getvalue(), "")
+
+class LaunchTor(unittest.TestCase):
+    def _do_test_launch(self, tor_executable):
+        reactor = object()
+        private_dir = "private"
+        txtorcon = mock.Mock()
+        tpp = mock.Mock
+        tpp.tor_protocol = mock.Mock()
+        txtorcon.launch_tor = mock.Mock(return_value=tpp)
+
+        with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
+                        return_value=999999):
+            d = tor_provider._launch_tor(reactor, tor_executable, private_dir,
+                                         txtorcon)
+        tor_control_endpoint, tor_control_proto = self.successResultOf(d)
+        self.assertIs(tor_control_proto, tpp.tor_protocol)
+
+    def test_launch(self):
+        return self._do_test_launch(None)
+    def test_launch_executable(self):
+        return self._do_test_launch("mytor")
+
+class ConnectToTor(unittest.TestCase):
+    def _do_test_connect(self, endpoint, reachable):
+        reactor = object()
+        txtorcon = object()
+        args = []
+        if endpoint:
+            args = ["--tor-control-port=%s" % endpoint]
+        cli_config = make_cli_config("basedir", "--listen=tor", *args)
+        stdout = cli_config.stdout
+        expected_port = "tcp:127.0.0.1:9151"
+        if endpoint:
+            expected_port = endpoint
+        tor_state = mock.Mock
+        tor_state.protocol = object()
+        tried = []
+        def _try_to_connect(reactor, port, stdout, txtorcon):
+            tried.append( (reactor, port, stdout, txtorcon) )
+            if not reachable:
+                return defer.succeed(None)
+            if port == expected_port: # second one on the list
+                return defer.succeed(tor_state)
+            return defer.succeed(None)
+
+        with mock.patch("allmydata.util.tor_provider._try_to_connect",
+                        _try_to_connect):
+            d = tor_provider._connect_to_tor(reactor, cli_config, txtorcon)
+        if not reachable:
+            f = self.failureResultOf(d)
+            self.assertIsInstance(f.value, ValueError)
+            self.assertEqual(str(f.value),
+                             "unable to reach any default Tor control port")
+            return
+        successful_port, tor_control_proto = self.successResultOf(d)
+        self.assertEqual(successful_port, expected_port)
+        self.assertIs(tor_control_proto, tor_state.protocol)
+        expected = [(reactor, "unix:/var/run/tor/control", stdout, txtorcon),
+                    (reactor, "tcp:127.0.0.1:9051", stdout, txtorcon),
+                    (reactor, "tcp:127.0.0.1:9151", stdout, txtorcon),
+                    ]
+        if endpoint:
+            expected = [(reactor, endpoint, stdout, txtorcon)]
+        self.assertEqual(tried, expected)
+
+    def test_connect(self):
+        return self._do_test_connect(None, True)
+    def test_connect_endpoint(self):
+        return self._do_test_connect("tcp:other:port", True)
+    def test_connect_unreachable(self):
+        return self._do_test_connect(None, False)
+
+
+class CreateOnion(unittest.TestCase):
+    def test_no_txtorcon(self):
+        with mock.patch("allmydata.util.tor_provider._import_txtorcon",
+                        return_value=None):
+            d = tor_provider.create_onion("reactor", "cli_config")
+            f = self.failureResultOf(d)
+            self.assertIsInstance(f.value, ValueError)
+            self.assertEqual(str(f.value),
+                             "Cannot create onion without txtorcon. "
+                             "Please 'pip install tahoe-lafs[tor]' to fix this.")
+    def _do_test_launch(self, executable):
+        basedir = self.mktemp()
+        os.mkdir(basedir)
+        private_dir = os.path.join(basedir, "private")
+        os.mkdir(private_dir)
+        reactor = object()
+        args = ["--listen=tor", "--tor-launch"]
+        if executable:
+            args.append("--tor-executable=%s" % executable)
+        cli_config = make_cli_config(basedir, *args)
+        protocol = object()
+        launch_tor = mock.Mock(return_value=defer.succeed(("control_endpoint",
+                                                           protocol)))
+        txtorcon = mock.Mock()
+        ehs = mock.Mock()
+        ehs.private_key = "privkey"
+        ehs.hostname = "ONION.onion"
+        txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
+        ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
+        ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
+
+        with mock_txtorcon(txtorcon):
+            with mock.patch("allmydata.util.tor_provider._launch_tor",
+                            launch_tor):
+                with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
+                                return_value=999999):
+                    d = tor_provider.create_onion(reactor, cli_config)
+        tahoe_config_tor, tor_port, tor_location = self.successResultOf(d)
+
+        launch_tor.assert_called_with(reactor, executable,
+                                      os.path.abspath(private_dir), txtorcon)
+        txtorcon.EphemeralHiddenService.assert_called_with("999999 127.0.0.1:3457")
+        ehs.add_to_tor.assert_called_with(protocol)
+        ehs.remove_from_tor.assert_called_with(protocol)
+
+        expected = {"launch": "true",
+                    "onion": "true",
+                    "onion.local_port": "999999",
+                    "onion.external_port": "3457",
+                    "onion.private_key_file": "private/tor_onion.privkey",
+                    }
+        if executable:
+            expected["tor.executable"] = executable
+        self.assertEqual(tahoe_config_tor, expected)
+        self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1")
+        self.assertEqual(tor_location, "tor:ONION.onion:3457")
+        fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"])
+        with open(fn, "rb") as f:
+            privkey = f.read()
+        self.assertEqual(privkey, "privkey")
+
+    def test_launch(self):
+        return self._do_test_launch(None)
+    def test_launch_executable(self):
+        return self._do_test_launch("mytor")
+
+    def test_control_endpoint(self):
+        basedir = self.mktemp()
+        os.mkdir(basedir)
+        private_dir = os.path.join(basedir, "private")
+        os.mkdir(private_dir)
+        reactor = object()
+        cli_config = make_cli_config(basedir, "--listen=tor")
+        protocol = object()
+        connect_to_tor = mock.Mock(return_value=defer.succeed(("goodport",
+                                                               protocol)))
+        txtorcon = mock.Mock()
+        ehs = mock.Mock()
+        ehs.private_key = "privkey"
+        ehs.hostname = "ONION.onion"
+        txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
+        ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
+        ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
+
+        with mock_txtorcon(txtorcon):
+            with mock.patch("allmydata.util.tor_provider._connect_to_tor",
+                            connect_to_tor):
+                with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
+                                return_value=999999):
+                    d = tor_provider.create_onion(reactor, cli_config)
+        tahoe_config_tor, tor_port, tor_location = self.successResultOf(d)
+
+        connect_to_tor.assert_called_with(reactor, cli_config, txtorcon)
+        txtorcon.EphemeralHiddenService.assert_called_with("999999 127.0.0.1:3457")
+        ehs.add_to_tor.assert_called_with(protocol)
+        ehs.remove_from_tor.assert_called_with(protocol)
+
+        expected = {"control.port": "goodport",
+                    "onion": "true",
+                    "onion.local_port": "999999",
+                    "onion.external_port": "3457",
+                    "onion.private_key_file": "private/tor_onion.privkey",
+                    }
+        self.assertEqual(tahoe_config_tor, expected)
+        self.assertEqual(tor_port, "tcp:999999:interface=127.0.0.1")
+        self.assertEqual(tor_location, "tor:ONION.onion:3457")
+        fn = os.path.join(basedir, tahoe_config_tor["onion.private_key_file"])
+        with open(fn, "rb") as f:
+            privkey = f.read()
+        self.assertEqual(privkey, "privkey")
+
+_None = object()
+class FakeConfig(dict):
+    def get_config(self, section, option, default=_None, boolean=False):
+        if section != "tor":
+            raise ValueError(section)
+        value = self.get(option, default)
+        if value is _None:
+            raise KeyError
+        return value
+
+class Provider(unittest.TestCase):
+    def test_build(self):
+        tor_provider.Provider("basedir", FakeConfig(), "reactor")
+
+    def test_handler_disabled(self):
+        p = tor_provider.Provider("basedir", FakeConfig(enabled=False),
+                                  "reactor")
+        self.assertEqual(p.get_tor_handler(), None)
+
+    def test_handler_no_tor(self):
+        with mock_tor(None):
+            p = tor_provider.Provider("basedir", FakeConfig(), "reactor")
+        self.assertEqual(p.get_tor_handler(), None)
+
+    def test_handler_launch_no_txtorcon(self):
+        with mock_txtorcon(None):
+            p = tor_provider.Provider("basedir", FakeConfig(launch=True),
+                                      "reactor")
+        self.assertEqual(p.get_tor_handler(), None)
+
+    @defer.inlineCallbacks
+    def test_handler_launch(self):
+        reactor = object()
+        tor = mock.Mock()
+        txtorcon = mock.Mock()
+        handler = object()
+        tor.control_endpoint_maker = mock.Mock(return_value=handler)
+        with mock_tor(tor):
+            with mock_txtorcon(txtorcon):
+                p = tor_provider.Provider("basedir", FakeConfig(launch=True),
+                                          reactor)
+        h = p.get_tor_handler()
+        self.assertIs(h, handler)
+        tor.control_endpoint_maker.assert_called_with(p._make_control_endpoint)
+
+        # make sure Tor is launched just once, the first time an endpoint is
+        # requested, and never again. The clientFromString() function is
+        # called once each time.
+
+        ep_desc = object()
+        launch_tor = mock.Mock(return_value=defer.succeed((ep_desc,None)))
+        ep = object()
+        cfs = mock.Mock(return_value=ep)
+        with mock.patch("allmydata.util.tor_provider._launch_tor", launch_tor):
+            with mock.patch("allmydata.util.tor_provider.clientFromString", cfs):
+                d = p._make_control_endpoint(reactor)
+                yield flushEventualQueue()
+                self.assertIs(self.successResultOf(d), ep)
+                launch_tor.assert_called_with(reactor, None, "basedir/private",
+                                              txtorcon)
+                cfs.assert_called_with(reactor, ep_desc)
+
+        launch_tor2 = mock.Mock(return_value=defer.succeed((ep_desc,None)))
+        cfs2 = mock.Mock(return_value=ep)
+        with mock.patch("allmydata.util.tor_provider._launch_tor", launch_tor2):
+            with mock.patch("allmydata.util.tor_provider.clientFromString", cfs2):
+                d2 = p._make_control_endpoint(reactor)
+                yield flushEventualQueue()
+                self.assertIs(self.successResultOf(d2), ep)
+                self.assertEqual(launch_tor2.mock_calls, [])
+                cfs2.assert_called_with(reactor, ep_desc)
+
+    def test_handler_socks_endpoint(self):
+        tor = mock.Mock()
+        handler = object()
+        tor.socks_endpoint = mock.Mock(return_value=handler)
+        ep = object()
+        cfs = mock.Mock(return_value=ep)
+        reactor = object()
+
+        with mock_tor(tor):
+            p = tor_provider.Provider("basedir",
+                                      FakeConfig(**{"socks.port": "ep_desc"}),
+                                      reactor)
+            with mock.patch("allmydata.util.tor_provider.clientFromString", cfs):
+                h = p.get_tor_handler()
+        cfs.assert_called_with(reactor, "ep_desc")
+        tor.socks_endpoint.assert_called_with(ep)
+        self.assertIs(h, handler)
+
+    def test_handler_control_endpoint(self):
+        tor = mock.Mock()
+        handler = object()
+        tor.control_endpoint = mock.Mock(return_value=handler)
+        ep = object()
+        cfs = mock.Mock(return_value=ep)
+        reactor = object()
+
+        with mock_tor(tor):
+            p = tor_provider.Provider("basedir",
+                                      FakeConfig(**{"control.port": "ep_desc"}),
+                                      reactor)
+            with mock.patch("allmydata.util.tor_provider.clientFromString", cfs):
+                h = p.get_tor_handler()
+        self.assertIs(h, handler)
+        cfs.assert_called_with(reactor, "ep_desc")
+        tor.control_endpoint.assert_called_with(ep)
+
+    def test_handler_default(self):
+        tor = mock.Mock()
+        handler = object()
+        tor.default_socks = mock.Mock(return_value=handler)
+
+        with mock_tor(tor):
+            p = tor_provider.Provider("basedir", FakeConfig(), "reactor")
+            h = p.get_tor_handler()
+        self.assertIs(h, handler)
+        tor.default_socks.assert_called_with()
+
+class Provider_CheckOnionConfig(unittest.TestCase):
+    def test_default(self):
+        # default config doesn't start an onion service, so it should be
+        # happy both with and without txtorcon
+
+        p = tor_provider.Provider("basedir", FakeConfig(), "reactor")
+        p.check_onion_config()
+
+        with mock_txtorcon(None):
+            p = tor_provider.Provider("basedir", FakeConfig(), "reactor")
+            p.check_onion_config()
+
+    def test_no_txtorcon(self):
+        with mock_txtorcon(None):
+            p = tor_provider.Provider("basedir", FakeConfig(onion=True),
+                                      "reactor")
+            e = self.assertRaises(ValueError, p.check_onion_config)
+            self.assertEqual(str(e), "Cannot create onion without txtorcon. "
+                             "Please 'pip install tahoe-lafs[tor]' to fix.")
+
+    def test_no_launch_no_control(self):
+        p = tor_provider.Provider("basedir", FakeConfig(onion=True), "reactor")
+        e = self.assertRaises(ValueError, p.check_onion_config)
+        self.assertEqual(str(e), "[tor] onion = true, but we have neither "
+                         "launch=true nor control.port=")
+
+    def test_missing_keys(self):
+        p = tor_provider.Provider("basedir", FakeConfig(onion=True,
+                                                        launch=True), "reactor")
+        e = self.assertRaises(ValueError, p.check_onion_config)
+        self.assertEqual(str(e), "[tor] onion = true, "
+                         "but onion.local_port= is missing")
+
+        p = tor_provider.Provider("basedir",
+                                  FakeConfig(onion=True, launch=True,
+                                             **{"onion.local_port": "x",
+                                                }), "reactor")
+        e = self.assertRaises(ValueError, p.check_onion_config)
+        self.assertEqual(str(e), "[tor] onion = true, "
+                         "but onion.external_port= is missing")
+
+        p = tor_provider.Provider("basedir",
+                                  FakeConfig(onion=True, launch=True,
+                                             **{"onion.local_port": "x",
+                                                "onion.external_port": "y",
+                                                }), "reactor")
+        e = self.assertRaises(ValueError, p.check_onion_config)
+        self.assertEqual(str(e), "[tor] onion = true, "
+                         "but onion.private_key_file= is missing")
+
+    def test_ok(self):
+        p = tor_provider.Provider("basedir",
+                                  FakeConfig(onion=True, launch=True,
+                                             **{"onion.local_port": "x",
+                                                "onion.external_port": "y",
+                                                "onion.private_key_file": "z",
+                                                }), "reactor")
+        p.check_onion_config()
+
+class Provider_Service(unittest.TestCase):
+    def test_no_onion(self):
+        reactor = object()
+        p = tor_provider.Provider("basedir", FakeConfig(onion=False), reactor)
+        with mock.patch("allmydata.util.tor_provider.Provider._start_onion") as s:
+            p.startService()
+        self.assertEqual(s.mock_calls, [])
+        self.assertEqual(p.running, True)
+
+        p.stopService()
+        self.assertEqual(p.running, False)
+
+    @defer.inlineCallbacks
+    def test_launch(self):
+        basedir = self.mktemp()
+        os.mkdir(basedir)
+        fn = os.path.join(basedir, "keyfile")
+        with open(fn, "w") as f:
+            f.write("private key")
+        reactor = object()
+        cfg = FakeConfig(onion=True, launch=True,
+                         **{"onion.local_port": 123,
+                            "onion.external_port": 456,
+                            "onion.private_key_file": "keyfile",
+                            })
+
+        txtorcon = mock.Mock()
+        with mock_txtorcon(txtorcon):
+            p = tor_provider.Provider(basedir, cfg, reactor)
+        tor_state = mock.Mock()
+        tor_state.protocol = object()
+        ehs = mock.Mock()
+        ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
+        ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
+        txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
+        launch_tor = mock.Mock(return_value=defer.succeed((None,tor_state.protocol)))
+        with mock.patch("allmydata.util.tor_provider._launch_tor",
+                        launch_tor):
+            d = p.startService()
+            yield flushEventualQueue()
+        self.successResultOf(d)
+        self.assertIs(p._onion_ehs, ehs)
+        self.assertIs(p._onion_tor_control_proto, tor_state.protocol)
+        launch_tor.assert_called_with(reactor, None,
+                                      os.path.join(basedir, "private"), txtorcon)
+        txtorcon.EphemeralHiddenService.assert_called_with("123 127.0.0.1:456",
+                                                           "private key")
+        ehs.add_to_tor.assert_called_with(tor_state.protocol)
+
+        yield p.stopService()
+        ehs.remove_from_tor.assert_called_with(tor_state.protocol)
+
+    @defer.inlineCallbacks
+    def test_control_endpoint(self):
+        basedir = self.mktemp()
+        os.mkdir(basedir)
+        fn = os.path.join(basedir, "keyfile")
+        with open(fn, "w") as f:
+            f.write("private key")
+        reactor = object()
+        cfg = FakeConfig(onion=True,
+                         **{"control.port": "ep_desc",
+                            "onion.local_port": 123,
+                            "onion.external_port": 456,
+                            "onion.private_key_file": "keyfile",
+                            })
+
+        txtorcon = mock.Mock()
+        with mock_txtorcon(txtorcon):
+            p = tor_provider.Provider(basedir, cfg, reactor)
+        tor_state = mock.Mock()
+        tor_state.protocol = object()
+        txtorcon.build_tor_connection = mock.Mock(return_value=tor_state)
+        ehs = mock.Mock()
+        ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
+        ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
+        txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
+        tcep = object()
+        cfs = mock.Mock(return_value=tcep)
+        with mock.patch("allmydata.util.tor_provider.clientFromString", cfs):
+            d = p.startService()
+            yield flushEventualQueue()
+        self.successResultOf(d)
+        self.assertIs(p._onion_ehs, ehs)
+        self.assertIs(p._onion_tor_control_proto, tor_state.protocol)
+        cfs.assert_called_with(reactor, "ep_desc")
+        txtorcon.build_tor_connection.assert_called_with(tcep)
+        txtorcon.EphemeralHiddenService.assert_called_with("123 127.0.0.1:456",
+                                                           "private key")
+        ehs.add_to_tor.assert_called_with(tor_state.protocol)
+
+        yield p.stopService()
+        ehs.remove_from_tor.assert_called_with(tor_state.protocol)

src/allmydata/util/tor_provider.py

@@ -1,49 +1,303 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, with_statement
+import os
+from twisted.internet.defer import inlineCallbacks, returnValue
+from twisted.internet.endpoints import clientFromString
+from twisted.internet.error import ConnectionRefusedError, ConnectError
+from twisted.application import service
+from .observer import OneShotObserverList
+from .iputil import allocate_tcp_port
 
-from __future__ import absolute_import
+def _import_tor():
-from __future__ import print_function
+    # this exists to be overridden by unit tests
-from __future__ import with_statement
+    try:
+        from foolscap.connections import tor
+        return tor
+    except ImportError: # pragma: no cover
+        return None
 
-from twisted.internet import reactor, defer
+def _import_txtorcon():
+    try:
+        import txtorcon
+        return txtorcon
+    except ImportError: # pragma: no cover
+        return None
 
-import txtorcon
+def data_directory(private_dir):
-from txtorcon import torconfig
+    return os.path.join(private_dir, "tor-statedir")
-from txtorcon import torcontrolprotocol
 
-@defer.inlineCallbacks
+# different ways we might approach this:
-def CreateOnion(tor_provider, key_file, onion_port):
-    local_port = yield txtorcon.util.available_tcp_port(reactor)
-    # XXX in the future we need to make it use UNIX domain sockets instead of TCP
-    hs_string = '%s 127.0.0.1:%d' % (onion_port, local_port)
-    service = txtorcon.EphemeralHiddenService([hs_string])
-    tor_protocol = yield tor_provider.get_control_protocol()
-    yield service.add_to_tor(tor_protocol)
 
-class TorProvider:
+# 1: get an ITorControlProtocol, make a
-    def __init__(self, tor_binary=None, data_directory=None, control_endpoint=None):
+# txtorcon.EphemeralHiddenService(ports), yield ehs.add_to_tor(tcp), store
-        assert tor_binary is not None or control_endpoint is not None
+# ehs.hostname and ehs.private_key, yield ehs.remove_from_tor(tcp)
-        self.data_directory = data_directory
-        self.tor_binary = tor_binary
-        self.control_endpoint = control_endpoint
-        self.tor_control_protocol = None
 
-    def get_control_protocol(self):
+def _try_to_connect(reactor, endpoint_desc, stdout, txtorcon):
-        """
+    # yields a TorState, or None
-        Returns a deferred which fires with the txtorcon tor control port object
+    ep = clientFromString(reactor, endpoint_desc)
-        """
+    d = txtorcon.build_tor_connection(ep)
-        if self.tor_control_protocol is not None:
+    def _failed(f):
-            return defer.succeed(self.tor_control_protocol)
+        # depending upon what's listening at that endpoint, we might get
+        # various errors. If this list is too short, we might expose an
+        # exception to the user (causing "tahoe create-node" to fail messily)
+        # when we're supposed to just try the next potential port instead.
+        # But I don't want to catch everything, because that may hide actual
+        # coding errrors.
+        f.trap(ConnectionRefusedError, # nothing listening on TCP
+               ConnectError, # missing unix socket, or permission denied
+               #ValueError,
+               # connecting to e.g. an HTTP server causes an
+               # UnhandledException (around a ValueError) when the handshake
+               # fails to parse, but that's not something we can catch. The
+               # attempt hangs, so don't do that.
+               RuntimeError, # authentication failure
+               )
+        if stdout:
+            stdout.write("Unable to reach Tor at '%s': %s\n" %
+                         (endpoint_desc, f.value))
+        return None
+    d.addErrback(_failed)
+    return d
+
+@inlineCallbacks
+def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
+    # TODO: handle default tor-executable
+    # TODO: it might be a good idea to find exactly which Tor we used,
+    # and record it's absolute path into tahoe.cfg . This would protect
+    # us against one Tor being on $PATH at create-node time, but then a
+    # different Tor being present at node startup. OTOH, maybe we don't
+    # need to worry about it.
+    tor_config = txtorcon.TorConfig()
+    tor_config.DataDirectory = data_directory(private_dir)
+
+    if True: # unix-domain control socket
+        tor_config.ControlPort = os.path.join(private_dir, "tor.control")
+        tor_control_endpoint_desc = "unix:%s" % tor_config.ControlPort
+    else:
+        # we allocate a new TCP control port each time
+        tor_config.ControlPort = allocate_tcp_port()
+        tor_control_endpoint_desc = "tcp:127.0.0.1:%d" % tor_config.ControlPort
+
+    tpp = yield txtorcon.launch_tor(tor_config, reactor,
+                                    tor_binary=tor_executable)
+    # now tor is launched and ready to be spoken to
+    # as a side effect, we've got an ITorControlProtocol ready to go
+    tor_control_proto = tpp.tor_protocol
+
+    # How/when to shut down the new process? for normal usage, the child
+    # tor will exit when it notices its parent (us) quit. Unit tests will
+    # mock out txtorcon.launch_tor(), so there will never be a real Tor
+    # process. So I guess we don't need to track the process.
+
+    returnValue((tor_control_endpoint_desc, tor_control_proto))
+
+@inlineCallbacks
+def _connect_to_tor(reactor, cli_config, txtorcon):
+    # we assume tor is already running
+    ports_to_try = ["unix:/var/run/tor/control",
+                    "tcp:127.0.0.1:9051",
+                    "tcp:127.0.0.1:9151", # TorBrowserBundle
+                    ]
+    if cli_config["tor-control-port"]:
+        ports_to_try = [cli_config["tor-control-port"]]
+    for port in ports_to_try:
+        tor_state = yield _try_to_connect(reactor, port, cli_config.stdout,
+                                          txtorcon)
+        if tor_state:
+            tor_control_proto = tor_state.protocol
+            returnValue((port, tor_control_proto)) ; break # helps editor
+    else:
+        raise ValueError("unable to reach any default Tor control port")
+
+@inlineCallbacks
+def create_onion(reactor, cli_config):
+    txtorcon = _import_txtorcon()
+    if not txtorcon:
+        raise ValueError("Cannot create onion without txtorcon. "
+                         "Please 'pip install tahoe-lafs[tor]' to fix this.")
+    tahoe_config_tor = {} # written into tahoe.cfg:[tor]
+    private_dir = os.path.abspath(os.path.join(cli_config["basedir"], "private"))
+    stdout = cli_config.stdout
+    if cli_config["tor-launch"]:
+        tahoe_config_tor["launch"] = "true"
+        tor_executable = cli_config["tor-executable"]
+        if tor_executable:
+            tahoe_config_tor["tor.executable"] = tor_executable
+        print("launching Tor (to allocate .onion address)..", file=stdout)
+        (_, tor_control_proto) = yield _launch_tor(
+            reactor, tor_executable, private_dir, txtorcon)
+        print("Tor launched", file=stdout)
+    else:
+        print("connecting to Tor (to allocate .onion address)..", file=stdout)
+        (port, tor_control_proto) = yield _connect_to_tor(
+            reactor, cli_config, txtorcon)
+        print("Tor connection established", file=stdout)
+        tahoe_config_tor["control.port"] = port
+
+    external_port = 3457 # TODO: pick this randomly? there's no contention.
+
+    local_port = allocate_tcp_port()
+    ehs = txtorcon.EphemeralHiddenService("%d 127.0.0.1:%d" %
+                                          (local_port, external_port))
+    print("allocating .onion address (takes ~40s)..", file=stdout)
+    yield ehs.add_to_tor(tor_control_proto)
+    print(".onion address allocated", file=stdout)
+    tor_port = "tcp:%d:interface=127.0.0.1" % local_port
+    tor_location = "tor:%s:%d" % (ehs.hostname, external_port)
+    privkey = ehs.private_key
+    yield ehs.remove_from_tor(tor_control_proto)
+
+    # in addition to the "how to launch/connect-to tor" keys above, we also
+    # record information about the onion service into tahoe.cfg.
+    # * "local_port" is a server endpont string, which should match
+    #   "tor_port" (which will be added to tahoe.cfg [node] tub.port)
+    # * "external_port" is the random "public onion port" (integer), which
+    #   (when combined with the .onion address) should match "tor_location"
+    #   (which will be added to tub.location)
+    # * "private_key_file" points to the on-disk copy of the private key
+    #   material (although we always write it to the same place)
+
+    tahoe_config_tor["onion"] = "true"
+    tahoe_config_tor["onion.local_port"] = str(local_port)
+    tahoe_config_tor["onion.external_port"] = str(external_port)
+    assert privkey
+    tahoe_config_tor["onion.private_key_file"] = os.path.join("private",
+                                                              "tor_onion.privkey")
+    privkeyfile = os.path.join(private_dir, "tor_onion.privkey")
+    with open(privkeyfile, "wb") as f:
+        f.write(privkey)
+
+    # tahoe_config_tor: this is a dictionary of keys/values to add to the
+    # "[tor]" section of tahoe.cfg, which tells the new node how to launch
+    # Tor in the right way.
+
+    # tor_port: a server endpoint string, it will be added to tub.port=
+
+    # tor_location: a foolscap connection hint, "tor:ONION:EXTERNAL_PORT"
+
+    # We assume/require that the Node gives us the same data_directory=
+    # at both create-node and startup time. The data directory is not
+    # recorded in tahoe.cfg
+
+    returnValue((tahoe_config_tor, tor_port, tor_location))
+
+# we can always create a Provider. If foolscap.connections.tor or txtorcon
+# are not installed, then get_tor_handler() will return None. If tahoe.cfg
+# wants to start an onion service too, then check_onion_config() will throw a
+# nice error, and startService will throw an ugly error.
+
+class Provider(service.MultiService):
+    def __init__(self, basedir, node_for_config, reactor):
+        service.MultiService.__init__(self)
+        self._basedir = basedir
+        self._node_for_config = node_for_config
+        self._tor_launched = None
+        self._onion_ehs = None
+        self._onion_tor_control_proto = None
+        self._tor = _import_tor()
+        self._txtorcon = _import_txtorcon()
+        self._reactor = reactor
+
+    def _get_tor_config(self, *args, **kwargs):
+        return self._node_for_config.get_config("tor", *args, **kwargs)
+
+    def get_tor_handler(self):
+        enabled = self._get_tor_config("enabled", True, boolean=True)
+        if not enabled:
+            return None
+        if not self._tor:
+            return None
+
+        if self._get_tor_config("launch", False, boolean=True):
+            if not self._txtorcon:
+                return None
+            return self._tor.control_endpoint_maker(self._make_control_endpoint)
+
+        socks_endpoint_desc = self._get_tor_config("socks.port", None)
+        if socks_endpoint_desc:
+            socks_ep = clientFromString(self._reactor, socks_endpoint_desc)
+            return self._tor.socks_endpoint(socks_ep)
+
+        controlport = self._get_tor_config("control.port", None)
+        if controlport:
+            ep = clientFromString(self._reactor, controlport)
+            return self._tor.control_endpoint(ep)
+
+        return self._tor.default_socks()
+
+    @inlineCallbacks
+    def _make_control_endpoint(self, reactor):
+        # this will only be called when tahoe.cfg has "[tor] launch = true"
+        (endpoint_desc, _) = yield self._get_launched_tor(reactor)
+        tor_control_endpoint = clientFromString(reactor, endpoint_desc)
+        returnValue(tor_control_endpoint)
+
+    def _get_launched_tor(self, reactor):
+        # this fires with a tuple of (control_endpoint, tor_protocol)
+        if not self._tor_launched:
+            self._tor_launched = OneShotObserverList()
+            private_dir = os.path.join(self._basedir, "private")
+            tor_binary = self._get_tor_config("tor.executable", None)
+            d = _launch_tor(reactor, tor_binary, private_dir, self._txtorcon)
+            d.addBoth(self._tor_launched.fire)
+        return self._tor_launched.when_fired()
+
+    def check_onion_config(self):
+        if self._get_tor_config("onion", False, boolean=True):
+            if not self._txtorcon:
+                raise ValueError("Cannot create onion without txtorcon. "
+                                 "Please 'pip install tahoe-lafs[tor]' to fix.")
+
+            # to start an onion server, we either need a Tor control port, or
+            # we need to launch tor
+            launch = self._get_tor_config("launch", False, boolean=True)
+            controlport = self._get_tor_config("control.port", None)
+            if not launch and not controlport:
+                raise ValueError("[tor] onion = true, but we have neither "
+                                 "launch=true nor control.port=")
+            # check that all the expected onion-specific keys are present
+            def require(name):
+                if not self._get_tor_config("onion.%s" % name, None):
+                    raise ValueError("[tor] onion = true,"
+                                     " but onion.%s= is missing" % name)
+            require("local_port")
+            require("external_port")
+            require("private_key_file")
+
+    @inlineCallbacks
+    def _start_onion(self, reactor):
+        # launch tor, if necessary
+        if self._get_tor_config("launch", False, boolean=True):
+            (_, tor_control_proto) = yield self._get_launched_tor(reactor)
         else:
-            if self.control_endpoint is None:
+            controlport = self._get_tor_config("control.port", None)
-                config = torconfig.TorConfig()
+            tcep = clientFromString(reactor, controlport)
-                if self.data_directory is not None:
+            tor_state = yield self._txtorcon.build_tor_connection(tcep)
-                    config['DataDirectory'] = self.data_directory
+            tor_control_proto = tor_state.protocol
-                d = torconfig.launch_tor(config, reactor, tor_binary=self.tor_binary)
+
-                d.addCallback(lambda result: result.tor_protocol)
+        local_port = int(self._get_tor_config("onion.local_port"))
-            else:
+        external_port = int(self._get_tor_config("onion.external_port"))
-                d = torcontrolprotocol.connect(self.control_endpoint)
+
-            def remember_tor_protocol(result):
+        fn = self._get_tor_config("onion.private_key_file")
-                self.tor_control_protocol = result
+        privkeyfile = os.path.join(self._basedir, fn)
-                return result
+        with open(privkeyfile, "rb") as f:
-            d.addCallback(remember_tor_protocol)
+            privkey = f.read()
-        return d
+        ehs = self._txtorcon.EphemeralHiddenService(
+            "%d 127.0.0.1:%d" % (local_port, external_port), privkey)
+        yield ehs.add_to_tor(tor_control_proto)
+        self._onion_ehs = ehs
+        self._onion_tor_control_proto = tor_control_proto
+
+
+    def startService(self):
+        service.MultiService.startService(self)
+        # if we need to start an onion service, now is the time
+        if self._get_tor_config("onion", False, boolean=True):
+            return self._start_onion(self._reactor) # so tests can synchronize
+
+    @inlineCallbacks
+    def stopService(self):
+        if self._onion_ehs and self._onion_tor_control_proto:
+            yield self._onion_ehs.remove_from_tor(self._onion_tor_control_proto)
+        # TODO: can we also stop tor?
+        yield service.MultiService.stopService(self)