diff --git a/docs/configuration.rst b/docs/configuration.rst index ab4751a04..0aab9b395 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -398,13 +398,13 @@ This section controls *when* Tor and I2P are used. The ``[tor]`` and ``[i2p]`` sections (described later) control *how* Tor/I2P connections are managed. -All Tahoe nodes need to make a connection to the Introducer; the ``[client] -introducer.furl`` setting (described below) indicates where the Introducer -lives. Tahoe client nodes must also make connections to storage servers: -these targets are specified in announcements that come from the Introducer. -Both are expressed as FURLs (a Foolscap URL), which include a list of -"connection hints". Each connection hint describes one (of perhaps many) -network endpoints where the service might live. +All Tahoe nodes need to make a connection to the Introducer; the +``private/introducers.yaml`` file (described below) configures where one or more +Introducers live. Tahoe client nodes must also make connections to storage +servers: these targets are specified in announcements that come from the +Introducer. Both are expressed as FURLs (a Foolscap URL), which include a +list of "connection hints". Each connection hint describes one (of perhaps +many) network endpoints where the service might live. Connection hints include a type, and look like: @@ -580,6 +580,8 @@ Client Configuration ``introducer.furl = (FURL string, mandatory)`` + DEPRECATED. See :ref:`introducer-definitions`. + This FURL tells the client how to connect to the introducer. Each Tahoe-LAFS grid is defined by an introducer. The introducer's FURL is created by the introducer node and written into its private base @@ -965,29 +967,28 @@ This section describes these other files. with as many people as possible, put the empty string (so that ``private/convergence`` is a zero-length file). -Additional Introducer Definitions -================================= +.. _introducer-definitions: -The ``private/introducers.yaml`` file defines additional Introducers. The -first introducer is defined in ``tahoe.cfg``, in ``[client] -introducer.furl``. To use two or more Introducers, choose a locally-unique -"petname" for each one, then define their FURLs in -``private/introducers.yaml`` like this:: +Introducer Definitions +====================== + +The ``private/introducers.yaml`` file defines Introducers. +Choose a locally-unique "petname" for each one then define their FURLs in ``private/introducers.yaml`` like this:: introducers: petname2: - furl: FURL2 + furl: "FURL2" petname3: - furl: FURL3 + furl: "FURL3" Servers will announce themselves to all configured introducers. Clients will merge the announcements they receive from all introducers. Nothing will re-broadcast an announcement (i.e. telling introducer 2 about something you heard from introducer 1). -If you omit the introducer definitions from both ``tahoe.cfg`` and -``introducers.yaml``, the node will not use an Introducer at all. Such -"introducerless" clients must be configured with static servers (described +If you omit the introducer definitions from ``introducers.yaml``, +the node will not use an Introducer at all. +Such "introducerless" clients must be configured with static servers (described below), or they will not be able to upload and download files. Static Server Definitions @@ -1152,7 +1153,6 @@ a legal one. timeout.disconnect = 1800 [client] - introducer.furl = pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo helper.furl = pb://ggti5ssoklj4y7eok5c3xkmj@tcp:helper.tahoe.example:7054/kk8lhr [storage] @@ -1163,6 +1163,11 @@ a legal one. [helper] enabled = True +To be introduced to storage servers, here is a sample ``private/introducers.yaml`` which can be used in conjunction:: + + introducers: + examplegrid: + furl: "pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo" Old Configuration Files ======================= diff --git a/docs/historical/configuration.rst b/docs/historical/configuration.rst index 660bc8489..7d9b9fbe4 100644 --- a/docs/historical/configuration.rst +++ b/docs/historical/configuration.rst @@ -20,7 +20,7 @@ Config setting File Comment ``[node]log_gatherer.furl`` ``BASEDIR/log_gatherer.furl`` (one per line) ``[node]timeout.keepalive`` ``BASEDIR/keepalive_timeout`` ``[node]timeout.disconnect`` ``BASEDIR/disconnect_timeout`` -``[client]introducer.furl`` ``BASEDIR/introducer.furl`` + ``BASEDIR/introducer.furl`` ``BASEDIR/private/introducers.yaml`` ``[client]helper.furl`` ``BASEDIR/helper.furl`` ``[client]key_generator.furl`` ``BASEDIR/key_generator.furl`` ``[client]stats_gatherer.furl`` ``BASEDIR/stats_gatherer.furl`` diff --git a/docs/running.rst b/docs/running.rst index 2b43adf75..ef6ba42ed 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -65,9 +65,9 @@ Running a Client To construct a client node, run “``tahoe create-client``”, which will create ``~/.tahoe`` to be the node's base directory. Acquire the ``introducer.furl`` (see below if you are running your own introducer, or use the one from the -`TestGrid page`_), and paste it after ``introducer.furl =`` in the -``[client]`` section of ``~/.tahoe/tahoe.cfg``. Then use “``tahoe run -~/.tahoe``”. After that, the node should be off and running. The first thing +`TestGrid page`_), and write it to ``~/.tahoe/private/introducers.yaml`` +(see :ref:`introducer-definitions`). Then use “``tahoe run ~/.tahoe``”. +After that, the node should be off and running. The first thing it will do is connect to the introducer and get itself connected to all other nodes on the grid. diff --git a/integration/test_tor.py b/integration/test_tor.py index 3d169a88f..dcbfb1151 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -1,7 +1,6 @@ from __future__ import print_function import sys -from os import mkdir from os.path import join import pytest @@ -9,6 +8,14 @@ import pytest_twisted import util +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.test.common import ( + write_introducer, +) + # see "conftest.py" for the fixtures (e.g. "tor_network") # XXX: Integration tests that involve Tor do not run reliably on @@ -66,12 +73,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne @pytest_twisted.inlineCallbacks def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl): - node_dir = join(temp_dir, name) + node_dir = FilePath(temp_dir).child(name) web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: - print("creating", node_dir) - mkdir(node_dir) + print("creating", node_dir.path) + node_dir.makedirs() proto = util._DumpOutputProtocol(None) reactor.spawnProcess( proto, @@ -84,12 +91,15 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ '--hide-ip', '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', - node_dir, + node_dir.path, ) ) yield proto.done - with open(join(node_dir, 'tahoe.cfg'), 'w') as f: + + # Which services should this client connect to? + write_introducer(node_dir, "default", introducer_furl) + with node_dir.child('tahoe.cfg').open('w') as f: f.write(''' [node] nickname = %(name)s @@ -105,15 +115,12 @@ onion = true onion.private_key_file = private/tor_onion.privkey [client] -# Which services should this client connect to? -introducer.furl = %(furl)s shares.needed = 1 shares.happy = 1 shares.total = 2 ''' % { 'name': name, - 'furl': introducer_furl, 'web_port': web_port, 'log_furl': flog_gatherer, 'control_port': control_port, @@ -121,5 +128,5 @@ shares.total = 2 }) print("running") - yield util._run_node(reactor, node_dir, request, None) + yield util._run_node(reactor, node_dir.path, request, None) print("okay, launched") diff --git a/newsfragments/3504.configuration b/newsfragments/3504.configuration new file mode 100644 index 000000000..9ff74482c --- /dev/null +++ b/newsfragments/3504.configuration @@ -0,0 +1 @@ +The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file. \ No newline at end of file diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 388757790..22952fb6d 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -3,7 +3,6 @@ from past.builtins import unicode import os, stat, time, weakref from base64 import urlsafe_b64encode from functools import partial -from errno import ENOENT, EPERM # On Python 2 this will be the backported package: from configparser import NoSectionError @@ -467,56 +466,17 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None): # we return this list introducer_clients = [] - introducers_yaml_filename = config.get_private_path("introducers.yaml") - introducers_filepath = FilePath(introducers_yaml_filename) + introducers = config.get_introducer_configuration() - try: - with introducers_filepath.open() as f: - introducers_yaml = yamlutil.safe_load(f) - if introducers_yaml is None: - raise EnvironmentError( - EPERM, - "Can't read '{}'".format(introducers_yaml_filename), - introducers_yaml_filename, - ) - introducers = introducers_yaml.get("introducers", {}) - log.msg( - "found {} introducers in private/introducers.yaml".format( - len(introducers), - ) - ) - except EnvironmentError as e: - if e.errno != ENOENT: - raise - introducers = {} - - if "default" in introducers.keys(): - raise ValueError( - "'default' introducer furl cannot be specified in introducers.yaml;" - " please fix impossible configuration." - ) - - # read furl from tahoe.cfg - tahoe_cfg_introducer_furl = config.get_config("client", "introducer.furl", None) - if tahoe_cfg_introducer_furl == "None": - raise ValueError( - "tahoe.cfg has invalid 'introducer.furl = None':" - " to disable it, use 'introducer.furl ='" - " or omit the key entirely" - ) - if tahoe_cfg_introducer_furl: - introducers[u'default'] = {'furl':tahoe_cfg_introducer_furl} - - for petname, introducer in introducers.items(): - introducer_cache_filepath = FilePath(config.get_private_path("introducer_{}_cache.yaml".format(petname))) + for petname, (furl, cache_path) in introducers.items(): ic = _introducer_factory( main_tub, - introducer['furl'].encode("ascii"), + furl.encode("ascii"), config.nickname, str(allmydata.__full_version__), str(_Client.OLDEST_SUPPORTED_VERSION), partial(_sequencer, config), - introducer_cache_filepath, + cache_path, ) introducer_clients.append(ic) return introducer_clients diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 54219a706..0fe62c50b 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -20,6 +20,8 @@ import re import types import errno from base64 import b32decode, b32encode +from errno import ENOENT, EPERM +from warnings import warn import attr @@ -41,6 +43,9 @@ from allmydata.util import fileutil, iputil from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import get_filesystem_encoding, quote_output from allmydata.util import configutil +from allmydata.util.yamlutil import ( + safe_load, +) from . import ( __full_version__, @@ -482,6 +487,97 @@ class _Config(object): os.path.join(self._basedir, *args) ) + def get_introducer_configuration(self): + """ + Get configuration for introducers. + + :return {unicode: (unicode, FilePath)}: A mapping from introducer + petname to a tuple of the introducer's fURL and local cache path. + """ + introducers_yaml_filename = self.get_private_path("introducers.yaml") + introducers_filepath = FilePath(introducers_yaml_filename) + + def get_cache_filepath(petname): + return FilePath( + self.get_private_path("introducer_{}_cache.yaml".format(petname)), + ) + + try: + with introducers_filepath.open() as f: + introducers_yaml = safe_load(f) + if introducers_yaml is None: + raise EnvironmentError( + EPERM, + "Can't read '{}'".format(introducers_yaml_filename), + introducers_yaml_filename, + ) + introducers = { + petname: config["furl"] + for petname, config + in introducers_yaml.get("introducers", {}).items() + } + non_strs = list( + k + for k + in introducers.keys() + if not isinstance(k, str) + ) + if non_strs: + raise TypeError( + "Introducer petnames {!r} should have been str".format( + non_strs, + ), + ) + non_strs = list( + v + for v + in introducers.values() + if not isinstance(v, str) + ) + if non_strs: + raise TypeError( + "Introducer fURLs {!r} should have been str".format( + non_strs, + ), + ) + log.msg( + "found {} introducers in {!r}".format( + len(introducers), + introducers_yaml_filename, + ) + ) + except EnvironmentError as e: + if e.errno != ENOENT: + raise + introducers = {} + + # supported the deprecated [client]introducer.furl item in tahoe.cfg + tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None) + if tahoe_cfg_introducer_furl == "None": + raise ValueError( + "tahoe.cfg has invalid 'introducer.furl = None':" + " to disable it omit the key entirely" + ) + if tahoe_cfg_introducer_furl: + warn( + "tahoe.cfg [client]introducer.furl is deprecated; " + "use private/introducers.yaml instead.", + category=DeprecationWarning, + stacklevel=-1, + ) + if "default" in introducers: + raise ValueError( + "'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml;" + " please fix impossible configuration." + ) + introducers['default'] = tahoe_cfg_introducer_furl + + return { + petname: (furl, get_cache_filepath(petname)) + for (petname, furl) + in introducers.items() + } + def create_tub_options(config): """ diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 34266ee72..b20cca65f 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,14 +4,15 @@ import os, sys, urllib, textwrap import codecs from os.path import join +from yaml import ( + safe_dump, +) + # Python 2 compatibility from future.utils import PY2 if PY2: from future.builtins import str # noqa: F401 -# On Python 2 this will be the backported package: -from configparser import NoSectionError - from twisted.python import usage from allmydata.util.assertutil import precondition @@ -115,24 +116,40 @@ class NoDefaultBasedirOptions(BasedirOptions): DEFAULT_ALIAS = u"tahoe" +def write_introducer(basedir, petname, furl): + """ + Overwrite the node's ``introducers.yaml`` with a file containing the given + introducer information. + """ + basedir.child(b"private").child(b"introducers.yaml").setContent( + safe_dump({ + "introducers": { + petname: { + "furl": furl.decode("ascii"), + }, + }, + }).encode("ascii"), + ) + + def get_introducer_furl(nodedir, config): """ :return: the introducer FURL for the given node (no matter if it's a client-type node or an introducer itself) """ + for petname, (furl, cache) in config.get_introducer_configuration().items(): + return furl + + # We have no configured introducers. Maybe this is running *on* the + # introducer? Let's guess, sure why not. try: - introducer_furl = config.get('client', 'introducer.furl') - except NoSectionError: - # we're not a client; maybe this is running *on* the introducer? - try: - with open(join(nodedir, "private", "introducer.furl"), "r") as f: - introducer_furl = f.read().strip() - except IOError: - raise Exception( - "Can't find introducer FURL in tahoe.cfg nor " - "{}/private/introducer.furl".format(nodedir) - ) - return introducer_furl + with open(join(nodedir, "private", "introducer.furl"), "r") as f: + return f.read().strip() + except IOError: + raise Exception( + "Can't find introducer FURL in tahoe.cfg nor " + "{}/private/introducer.furl".format(nodedir) + ) def get_aliases(nodedir): diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 2634e0915..a4b2213ed 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -5,11 +5,20 @@ import json from twisted.internet import reactor, defer from twisted.python.usage import UsageError -from allmydata.scripts.common import BasedirOptions, NoDefaultBasedirOptions +from twisted.python.filepath import ( + FilePath, +) + +from allmydata.scripts.common import ( + BasedirOptions, + NoDefaultBasedirOptions, + write_introducer, +) from allmydata.scripts.default_nodedir import _default_nodedir from allmydata.util.assertutil import precondition from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding from allmydata.util import fileutil, i2p_provider, iputil, tor_provider + from wormhole import wormhole @@ -299,12 +308,15 @@ def write_node_config(c, config): def write_client_config(c, config): - # note, config can be a plain dict, it seems -- see - # test_configutil.py in test_create_client_config + introducer = config.get("introducer", None) + if introducer is not None: + write_introducer( + FilePath(config["basedir"]), + "default", + introducer, + ) + c.write("[client]\n") - c.write("# Which services should this client connect to?\n") - introducer = config.get("introducer", None) or "" - c.write("introducer.furl = %s\n" % introducer) c.write("helper.furl =\n") c.write("#stats_gatherer.furl =\n") c.write("\n") @@ -437,8 +449,11 @@ def create_node(config): print("Node created in %s" % quote_local_unicode_path(basedir), file=out) tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg")) + introducers_yaml = quote_local_unicode_path( + os.path.join(basedir, "private", "introducers.yaml"), + ) if not config.get("introducer", ""): - print(" Please set [client]introducer.furl= in %s!" % tahoe_cfg, file=out) + print(" Please add introducers to %s!" % (introducers_yaml,), file=out) print(" The node cannot connect to a grid without it.", file=out) if not config.get("nickname", ""): print(" Please set [node]nickname= in %s" % tahoe_cfg, file=out) diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index cca4216e3..dbc84d0ea 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -1,16 +1,15 @@ from __future__ import print_function import json -from os.path import join from twisted.python import usage from twisted.internet import defer, reactor from wormhole import wormhole -from allmydata.util import configutil from allmydata.util.encodingutil import argv_to_abspath from allmydata.scripts.common import get_default_nodedir, get_introducer_furl +from allmydata.node import read_config class InviteOptions(usage.Options): @@ -77,7 +76,7 @@ def invite(options): basedir = argv_to_abspath(options.parent['node-directory']) else: basedir = get_default_nodedir() - config = configutil.get_config(join(basedir, 'tahoe.cfg')) + config = read_config(basedir, u"") out = options.stdout err = options.stderr diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 41cf6e1d7..6ec90eeae 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -8,6 +8,9 @@ if PY2: from future.builtins import str # noqa: F401 from six.moves import cStringIO as StringIO +from twisted.python.filepath import ( + FilePath, +) from twisted.internet import defer, reactor, protocol, error from twisted.application import service, internet from twisted.web import client as tw_client @@ -21,6 +24,10 @@ from allmydata.util import fileutil, pollmixin from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.util.encodingutil import get_filesystem_encoding +from allmydata.scripts.common import ( + write_introducer, +) + class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object): full_speed_ahead = False _bytes_so_far = 0 @@ -180,16 +187,18 @@ class SystemFramework(pollmixin.PollMixin): self.introducer_furl = self.introducer.introducer_url def make_nodes(self): + root = FilePath(self.testdir) self.nodes = [] for i in range(self.numnodes): - nodedir = os.path.join(self.testdir, "node%d" % i) - os.mkdir(nodedir) - f = open(os.path.join(nodedir, "tahoe.cfg"), "w") - f.write("[client]\n" - "introducer.furl = %s\n" - "shares.happy = 1\n" - "[storage]\n" - % (self.introducer_furl,)) + nodedir = root.child("node%d" % (i,)) + private = nodedir.child("private") + private.makedirs() + write_introducer(nodedir, "default", self.introducer_url) + config = ( + "[client]\n" + "shares.happy = 1\n" + "[storage]\n" + ) # the only tests for which we want the internal nodes to actually # retain shares are the ones where somebody's going to download # them. @@ -200,13 +209,13 @@ class SystemFramework(pollmixin.PollMixin): # for these tests, we tell the storage servers to pretend to # accept shares, but really just throw them out, since we're # only testing upload and not download. - f.write("debug_discard = true\n") + config += "debug_discard = true\n" if self.mode in ("receive",): # for this mode, the client-under-test gets all the shares, # so our internal nodes can refuse requests - f.write("readonly = true\n") - f.close() - c = client.Client(basedir=nodedir) + config += "readonly = true\n" + nodedir.child("tahoe.cfg").setContent(config) + c = client.Client(basedir=nodedir.path) c.setServiceParent(self) self.nodes.append(c) # the peers will start running, eventually they will connect to each @@ -235,16 +244,16 @@ this file are ignored. quiet = StringIO() create_node.create_node({'basedir': clientdir}, out=quiet) log.msg("DONE MAKING CLIENT") + write_introducer(clientdir, "default", self.introducer_furl) # now replace tahoe.cfg # set webport=0 and then ask the node what port it picked. f = open(os.path.join(clientdir, "tahoe.cfg"), "w") f.write("[node]\n" "web.port = tcp:0:interface=127.0.0.1\n" "[client]\n" - "introducer.furl = %s\n" "shares.happy = 1\n" "[storage]\n" - % (self.introducer_furl,)) + ) if self.mode in ("upload-self", "receive"): # accept and store shares, to trigger the memory consumption bugs diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py index 0daeb5840..f356e18de 100644 --- a/src/allmydata/test/cli/test_invite.py +++ b/src/allmydata/test/cli/test_invite.py @@ -8,7 +8,9 @@ from twisted.internet import defer from ..common_util import run_cli from ..no_network import GridTestMixin from .common import CLITestMixin - +from ...client import ( + read_config, +) class _FakeWormhole(object): @@ -81,9 +83,19 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase): ) self.assertEqual(0, rc) + + config = read_config(node_dir, u"") + self.assertIn( + "pb://foo", + set( + furl + for (furl, cache) + in config.get_introducer_configuration().values() + ), + ) + with open(join(node_dir, 'tahoe.cfg'), 'r') as f: config = f.read() - self.assertIn("pb://foo", config) self.assertIn(u"somethinghopefullyunique", config) @defer.inlineCallbacks diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index a420dd3ba..9476455f9 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -81,6 +81,9 @@ from allmydata.client import ( config_from_string, create_client_from_config, ) +from allmydata.scripts.common import ( + write_introducer, + ) from ..crypto import ( ed25519, @@ -222,7 +225,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib() - introducer_furl = attr.ib() + introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes)) node_config = attr.ib(default=attr.Factory(dict)) config = attr.ib(default=None) @@ -246,6 +249,11 @@ class UseNode(object): config=format_config_items(self.plugin_config), ) + write_introducer( + self.basedir, + "default", + self.introducer_furl, + ) self.config = config_from_string( self.basedir.asTextMode().path, "tub.port", @@ -254,11 +262,9 @@ class UseNode(object): {node_config} [client] -introducer.furl = {furl} storage.plugins = {storage_plugin} {plugin_config_section} """.format( - furl=self.introducer_furl, storage_plugin=self.storage_plugin, node_config=format_config_items(self.node_config), plugin_config_section=plugin_config_section, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 54c5be8e5..b0a6add2a 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -55,6 +55,9 @@ from allmydata.util import ( from allmydata.util.fileutil import abspath_expanduser_unicode from allmydata.interfaces import IFilesystemNode, IFileNode, \ IImmutableFileNode, IMutableFileNode, IDirectoryNode +from allmydata.scripts.common import ( + write_introducer, +) from foolscap.api import flushEventualQueue import allmydata.test.common_util as testutil from .common import ( @@ -72,13 +75,7 @@ from .matchers import ( SOME_FURL = b"pb://abcde@nowhere/fake" -BASECONFIG = ("[client]\n" - "introducer.furl = \n" - ) - -BASECONFIG_I = ("[client]\n" - "introducer.furl = %s\n" - ) +BASECONFIG = "[client]\n" class Basic(testutil.ReallyEqualMixin, unittest.TestCase): def test_loadable(self): @@ -120,14 +117,14 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): def write_config(s): config = ("[client]\n" - "introducer.furl = %s\n" % s) + "helper.furl = %s\n" % s) fileutil.write(os.path.join(basedir, "tahoe.cfg"), config) for s in should_fail: write_config(s) with self.assertRaises(UnescapedHashError) as ctx: yield client.create_client(basedir) - self.assertIn("[client]introducer.furl", str(ctx.exception)) + self.assertIn("[client]helper.furl", str(ctx.exception)) def test_unreadable_config(self): if sys.platform == "win32": @@ -665,12 +662,13 @@ class AnonymousStorage(SyncTestCase): """ If anonymous storage access is enabled then the client announces it. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() + write_introducer(basedir, "someintroducer", SOME_FURL) config = client.config_from_string( - basedir, + basedir.path, "tub.port", - BASECONFIG_I % (SOME_FURL,) + ( + BASECONFIG + ( "[storage]\n" "enabled = true\n" "anonymous = true\n" @@ -684,7 +682,7 @@ class AnonymousStorage(SyncTestCase): get_published_announcements(node), MatchesListwise([ matches_storage_announcement( - basedir, + basedir.path, anonymous=True, ), ]), @@ -696,12 +694,13 @@ class AnonymousStorage(SyncTestCase): If anonymous storage access is disabled then the client does not announce it nor does it write a fURL for it to beneath the node directory. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() + write_introducer(basedir, "someintroducer", SOME_FURL) config = client.config_from_string( - basedir, + basedir.path, "tub.port", - BASECONFIG_I % (SOME_FURL,) + ( + BASECONFIG + ( "[storage]\n" "enabled = true\n" "anonymous = false\n" @@ -715,7 +714,7 @@ class AnonymousStorage(SyncTestCase): get_published_announcements(node), MatchesListwise([ matches_storage_announcement( - basedir, + basedir.path, anonymous=False, ), ]), @@ -733,12 +732,12 @@ class AnonymousStorage(SyncTestCase): possible to reach the anonymous storage server via the originally published fURL. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() enabled_config = client.config_from_string( - basedir, + basedir.path, "tub.port", - BASECONFIG_I % (SOME_FURL,) + ( + BASECONFIG + ( "[storage]\n" "enabled = true\n" "anonymous = true\n" @@ -760,9 +759,9 @@ class AnonymousStorage(SyncTestCase): ) disabled_config = client.config_from_string( - basedir, + basedir.path, "tub.port", - BASECONFIG_I % (SOME_FURL,) + ( + BASECONFIG + ( "[storage]\n" "enabled = true\n" "anonymous = false\n" @@ -782,8 +781,8 @@ class IntroducerClients(unittest.TestCase): def test_invalid_introducer_furl(self): """ - An introducer.furl of 'None' is invalid and causes - create_introducer_clients to fail. + An introducer.furl of 'None' in the deprecated [client]introducer.furl + field is invalid and causes `create_introducer_clients` to fail. """ cfg = ( "[client]\n" @@ -948,20 +947,28 @@ class Run(unittest.TestCase, testutil.StallMixin): @defer.inlineCallbacks def test_loadable(self): - basedir = "test_client.Run.test_loadable" - os.mkdir(basedir) + """ + A configuration consisting only of an introducer can be turned into a + client node. + """ + basedir = FilePath("test_client.Run.test_loadable") + private = basedir.child("private") + private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" - fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG_I % dummy) - fileutil.write(os.path.join(basedir, client._Client.EXIT_TRIGGER_FILE), "") - yield client.create_client(basedir) + write_introducer(basedir, "someintroducer", dummy) + basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child(client._Client.EXIT_TRIGGER_FILE).touch() + yield client.create_client(basedir.path) @defer.inlineCallbacks def test_reloadable(self): - basedir = "test_client.Run.test_reloadable" - os.mkdir(basedir) + basedir = FilePath("test_client.Run.test_reloadable") + private = basedir.child("private") + private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" - fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG_I % dummy) - c1 = yield client.create_client(basedir) + write_introducer(basedir, "someintroducer", dummy) + basedir.child("tahoe.cfg").setContent(BASECONFIG) + c1 = yield client.create_client(basedir.path) c1.setServiceParent(self.sparent) # delay to let the service start up completely. I'm not entirely sure @@ -983,7 +990,7 @@ class Run(unittest.TestCase, testutil.StallMixin): # also change _check_exit_trigger to use it instead of a raw # reactor.stop, also instrument the shutdown event in an # attribute that we can check.) - c2 = yield client.create_client(basedir) + c2 = yield client.create_client(basedir.path) c2.setServiceParent(self.sparent) yield c2.disownServiceParent() @@ -1122,12 +1129,18 @@ class StorageAnnouncementTests(SyncTestCase): """ def setUp(self): super(StorageAnnouncementTests, self).setUp() - self.basedir = self.useFixture(TempDir()).path - create_node_dir(self.basedir, u"") + self.basedir = FilePath(self.useFixture(TempDir()).path) + create_node_dir(self.basedir.path, u"") + # Write an introducer configuration or we can't observer + # announcements. + write_introducer(self.basedir, "someintroducer", SOME_FURL) def get_config(self, storage_enabled, more_storage="", more_sections=""): return """ +[client] +# Empty + [node] tub.location = tcp:192.0.2.0:1234 @@ -1135,9 +1148,6 @@ tub.location = tcp:192.0.2.0:1234 enabled = {storage_enabled} {more_storage} -[client] -introducer.furl = pb://abcde@nowhere/fake - {more_sections} """.format( storage_enabled=storage_enabled, @@ -1151,7 +1161,7 @@ introducer.furl = pb://abcde@nowhere/fake No storage announcement is published if storage is not enabled. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config(storage_enabled=False), ) @@ -1173,7 +1183,7 @@ introducer.furl = pb://abcde@nowhere/fake storage is enabled. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config(storage_enabled=True), ) @@ -1190,7 +1200,7 @@ introducer.furl = pb://abcde@nowhere/fake # Match the following list (of one element) ... MatchesListwise([ # The only element in the list ... - matches_storage_announcement(self.basedir), + matches_storage_announcement(self.basedir.path), ]), )), ) @@ -1205,7 +1215,7 @@ introducer.furl = pb://abcde@nowhere/fake value = u"thing" config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1225,7 +1235,7 @@ introducer.furl = pb://abcde@nowhere/fake get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1246,7 +1256,7 @@ introducer.furl = pb://abcde@nowhere/fake self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1268,7 +1278,7 @@ introducer.furl = pb://abcde@nowhere/fake get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1294,7 +1304,7 @@ introducer.furl = pb://abcde@nowhere/fake self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1330,7 +1340,7 @@ introducer.furl = pb://abcde@nowhere/fake self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1346,7 +1356,7 @@ introducer.furl = pb://abcde@nowhere/fake get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1368,7 +1378,7 @@ introducer.furl = pb://abcde@nowhere/fake self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1395,7 +1405,7 @@ introducer.furl = pb://abcde@nowhere/fake available on the system. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 0b2f82f62..3aed3f049 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -52,8 +52,11 @@ from allmydata.util import pollmixin, idlib, fileutil, yamlutil from allmydata.util.iputil import ( listenOnUnused, ) +from allmydata.scripts.common import ( + write_introducer, +) import allmydata.test.common_util as testutil -from allmydata.test.common import ( +from .common import ( SyncTestCase, AsyncTestCase, AsyncBrokenTestCase, @@ -797,22 +800,28 @@ class Announcements(AsyncTestCase): @defer.inlineCallbacks def test_client_cache(self): - basedir = "introducer/ClientSeqnums/test_client_cache_1" - fileutil.make_dirs(basedir) - cache_filepath = FilePath(os.path.join(basedir, "private", - "introducer_default_cache.yaml")) + """ + Announcements received by an introducer client are written to that + introducer client's cache file. + """ + basedir = FilePath("introducer/ClientSeqnums/test_client_cache_1") + private = basedir.child("private") + private.makedirs() + write_introducer(basedir, "default", "nope") + cache_filepath = basedir.descendant([ + "private", + "introducer_default_cache.yaml", + ]) # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. - with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: - f.write("[client]\n") - f.write("introducer.furl = nope\n") + with basedir.child("tahoe.cfg").open("w") as f: f.write("[storage]\n") f.write("enabled = false\n") - c = yield create_client(basedir) + c = yield create_client(basedir.path) ic = c.introducer_clients[0] private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-") @@ -878,7 +887,7 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(ensure_binary(announcements[public_key_str2]["anonymous-storage-FURL"]), furl3) - c2 = yield create_client(basedir) + c2 = yield create_client(basedir.path) c2.introducer_clients[0]._load_announcements() yield flushEventualQueue() self.assertEqual(c2.storage_broker.get_all_serverids(), @@ -888,27 +897,24 @@ class ClientSeqnums(AsyncBrokenTestCase): @defer.inlineCallbacks def test_client(self): - basedir = "introducer/ClientSeqnums/test_client" - fileutil.make_dirs(basedir) + basedir = FilePath("introducer/ClientSeqnums/test_client") + private = basedir.child("private") + private.makedirs() + write_introducer(basedir, "default", "nope") # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. - f = open(os.path.join(basedir, "tahoe.cfg"), "w") - f.write("[client]\n") - f.write("introducer.furl = nope\n") - f.write("[storage]\n") - f.write("enabled = false\n") - f.close() + with basedir.child("tahoe.cfg").open("w") as f: + f.write("[storage]\n") + f.write("enabled = false\n") - c = yield create_client(basedir) + c = yield create_client(basedir.path) ic = c.introducer_clients[0] outbound = ic._outbound_announcements published = ic._published_announcements def read_seqnum(): - f = open(os.path.join(basedir, "announcement-seqnum")) - seqnum = f.read().strip() - f.close() + seqnum = basedir.child("announcement-seqnum").getContent() return int(seqnum) ic.publish("sA", {"key": "value1"}, c._node_private_key) diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py index 34e6e5d96..520a5a69a 100644 --- a/src/allmydata/test/test_multi_introducers.py +++ b/src/allmydata/test/test_multi_introducers.py @@ -24,9 +24,6 @@ class MultiIntroTests(unittest.TestCase): config = {'hide-ip':False, 'listen': 'tcp', 'port': None, 'location': None, 'hostname': 'example.net'} write_node_config(c, config) - fake_furl = "furl1" - c.write("[client]\n") - c.write("introducer.furl = %s\n" % fake_furl) c.write("[storage]\n") c.write("enabled = false\n") c.close() @@ -36,8 +33,10 @@ class MultiIntroTests(unittest.TestCase): @defer.inlineCallbacks def test_introducer_count(self): - """ Ensure that the Client creates same number of introducer clients - as found in "basedir/private/introducers" config file. """ + """ + If there are two introducers configured in ``introducers.yaml`` then + ``Client`` creates two introducer clients. + """ connections = { 'introducers': { u'intro1':{ 'furl': 'furl1' }, @@ -50,25 +49,13 @@ class MultiIntroTests(unittest.TestCase): ic_count = len(myclient.introducer_clients) # assertions - self.failUnlessEqual(ic_count, 3) - - @defer.inlineCallbacks - def test_introducer_count_commented(self): - """ Ensure that the Client creates same number of introducer clients - as found in "basedir/private/introducers" config file when there is one - commented.""" - self.yaml_path.setContent(INTRODUCERS_CFG_FURLS_COMMENTED) - # get a client and count of introducer_clients - myclient = yield create_client(self.basedir) - ic_count = len(myclient.introducer_clients) - - # assertions - self.failUnlessEqual(ic_count, 2) + self.failUnlessEqual(ic_count, len(connections["introducers"])) @defer.inlineCallbacks def test_read_introducer_furl_from_tahoecfg(self): - """ Ensure that the Client reads the introducer.furl config item from - the tahoe.cfg file. """ + """ + The deprecated [client]introducer.furl item is still read and respected. + """ # create a custom tahoe.cfg c = open(os.path.join(self.basedir, "tahoe.cfg"), "w") config = {'hide-ip':False, 'listen': 'tcp', @@ -87,20 +74,42 @@ class MultiIntroTests(unittest.TestCase): # assertions self.failUnlessEqual(fake_furl, tahoe_cfg_furl) + self.assertEqual( + list( + warning["message"] + for warning + in self.flushWarnings() + if warning["category"] is DeprecationWarning + ), + ["tahoe.cfg [client]introducer.furl is deprecated; " + "use private/introducers.yaml instead."], + ) @defer.inlineCallbacks def test_reject_default_in_yaml(self): - connections = {'introducers': { - u'default': { 'furl': 'furl1' }, - }} + """ + If an introducer is configured in tahoe.cfg with the deprecated + [client]introducer.furl then a "default" introducer in + introducers.yaml is rejected. + """ + connections = { + 'introducers': { + u'default': { 'furl': 'furl1' }, + }, + } self.yaml_path.setContent(yamlutil.safe_dump(connections)) + FilePath(self.basedir).child("tahoe.cfg").setContent( + "[client]\n" + "introducer.furl = furl1\n" + ) + with self.assertRaises(ValueError) as ctx: yield create_client(self.basedir) self.assertEquals( str(ctx.exception), - "'default' introducer furl cannot be specified in introducers.yaml; please " - "fix impossible configuration.", + "'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml; " + "please fix impossible configuration.", ) SIMPLE_YAML = """ @@ -126,8 +135,6 @@ class NoDefault(unittest.TestCase): config = {'hide-ip':False, 'listen': 'tcp', 'port': None, 'location': None, 'hostname': 'example.net'} write_node_config(c, config) - c.write("[client]\n") - c.write("# introducer.furl =\n") # omit default c.write("[storage]\n") c.write("enabled = false\n") c.close() diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index a4c64a654..be3be51b9 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -684,8 +684,6 @@ class TestMissingPorts(unittest.TestCase): BASE_CONFIG = """ -[client] -introducer.furl = empty [tor] enabled = false [i2p] diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..421c588ff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -458,7 +458,7 @@ class StoragePluginWebPresence(AsyncTestCase): }, storage_plugin=self.storage_plugin, basedir=self.basedir, - introducer_furl=ensure_text(SOME_FURL), + introducer_furl=SOME_FURL, )) self.node = yield self.node_fixture.create_node() self.webish = self.node.getServiceNamed(WebishServer.name) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 21da0a914..7a7fe117b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -33,6 +33,9 @@ from allmydata.mutable.publish import MutableData from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure +from twisted.python.filepath import ( + FilePath, +) from .common import ( TEST_RSA_KEY_SIZE, @@ -47,6 +50,9 @@ from .web.common import ( from allmydata.test.test_runner import RunBinTahoeMixin from . import common_util as testutil from .common_util import run_cli +from ..scripts.common import ( + write_introducer, +) LARGE_DATA = """ This is some data to publish to the remote grid.., which needs to be large @@ -806,8 +812,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): except1 = set(range(self.numclients)) - {1} feature_matrix = { - # client 1 uses private/introducers.yaml, not tahoe.cfg - ("client", "introducer.furl"): except1, ("client", "nickname"): except1, # client 1 has to auto-assign an address. @@ -833,7 +837,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): setnode = partial(setconf, config, which, "node") sethelper = partial(setconf, config, which, "helper") - setclient("introducer.furl", self.introducer_furl) setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) if self.stats_gatherer_furl: @@ -850,13 +853,11 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): sethelper("enabled", "True") - if which == 1: - # clients[1] uses private/introducers.yaml, not tahoe.cfg - iyaml = ("introducers:\n" - " petname2:\n" - " furl: %s\n") % self.introducer_furl - iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") - fileutil.write(iyaml_fn, iyaml) + iyaml = ("introducers:\n" + " petname2:\n" + " furl: %s\n") % self.introducer_furl + iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") + fileutil.write(iyaml_fn, iyaml) return _render_config(config) @@ -905,16 +906,21 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # usually this node is *not* parented to our self.sparent, so we can # shut it down separately from the rest, to exercise the # connection-lost code - basedir = self.getdir("client%d" % client_num) - if not os.path.isdir(basedir): - fileutil.make_dirs(basedir) + basedir = FilePath(self.getdir("client%d" % client_num)) + basedir.makedirs() config = "[client]\n" - config += "introducer.furl = %s\n" % self.introducer_furl if helper_furl: config += "helper.furl = %s\n" % helper_furl - fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) + basedir.child("tahoe.cfg").setContent(config) + private = basedir.child("private") + private.makedirs() + write_introducer( + basedir, + "default", + self.introducer_furl, + ) - c = yield client.create_client(basedir) + c = yield client.create_client(basedir.path) self.clients.append(c) c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) self.numclients += 1