diff --git a/docs/configuration.rst b/docs/configuration.rst index 2e9f216b1..92ca7e935 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -12,8 +12,9 @@ Configuring a Tahoe-LAFS node 6. `Running A Helper`_ 7. `Running An Introducer`_ 8. `Other Files in BASEDIR`_ -9. `Other files`_ -10. `Example`_ +9. `Static Server Definitions`_ +10. `Other files`_ +11. `Example`_ A Tahoe-LAFS node is configured by writing to files in its base directory. These files are read by the node when it starts, so each time you change @@ -662,6 +663,56 @@ This section describes these other files. ``private/convergence`` is a zero-length file). +Static Server Definitions +========================= + +The ``private/servers.yaml`` file defines "static servers": those which are +not announced through the Introducer. This can also control how we connect to +those servers. + +Most clients do not need this file. It is only necessary if you want to use +servers which are (for some specialized reason) not announced through the +Introducer, or to connect to those servers in different ways. You might do +this to "freeze" the server list: use the Introducer for a while, then copy +all announcements into ``servers.yaml``, then stop using the Introducer +entirely. Or you might have a private server that you don't want other users +to learn about (via the Introducer). Or you might run a local server which is +announced to everyone else as a Tor onion address, but which you can connect +to directly (via TCP). + +The file syntax is `YAML`_, with a top-level dictionary named ``storage``. +Other items may be added in the future. + +The ``storage`` dictionary takes keys which are server-ids, and values which +are dictionaries with two keys: ``ann`` and ``connections``. The ``ann`` +value is a dictionary which will be used in lieu of the introducer +announcement, so it can be populated by copying the ``ann`` dictionary from +``NODEDIR/introducer_cache.yaml``. Static servers which use the node's +default connection handlers only need a few keys: + +* the server ID, which can be any string +* a nickname, which is the string that is printed on the web interface +* the ``anonymous-storage-FURL``, which is where the server lives +* ``permutation-seed-base32``, which controls how shares are mapped to + servers. This is normally computed from the server-ID, but can be + overridden to maintain the mapping for older servers which used to use + Foolscap TubIDs as server-IDs. +* more important keys may be added in the future, as Accounting and + HTTP-based servers are implemented + +For example, a private static server could be defined with a +``private/servers.yaml`` file like this:: + + storage: + my-serverid-1: + ann: + nickname: my-server-1 + anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:8.8.8.8:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b + permutation-seed-base32: w2hqnbaa25yw4qgcvghl5psa3srpfgw3 + +.. _YAML: http://yaml.org/ + + Other files =========== diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d21e9faee..b96bcffb4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -8,7 +8,6 @@ from twisted.application import service from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from pycryptopp.publickey import rsa -from foolscap.api import eventually import allmydata from allmydata.storage.server import StorageServer @@ -128,7 +127,6 @@ class Client(node.Node, pollmixin.PollMixin): self.started_timestamp = time.time() self.logSource="Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() - self.load_connections() self.init_introducer_client() self.init_stats_provider() self.init_secrets() @@ -140,6 +138,7 @@ class Client(node.Node, pollmixin.PollMixin): if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") self.init_client() + self.load_static_servers() self.helper = None if self.get_config("helper", "enabled", False, boolean=True): self.init_helper() @@ -186,21 +185,6 @@ class Client(node.Node, pollmixin.PollMixin): self.introducer_client = ic ic.setServiceParent(self) - def load_connections(self): - """ - Load the connections.yaml file if it exists, otherwise - create a default configuration. - """ - fn = os.path.join(self.basedir, "private", "connections.yaml") - connections_filepath = FilePath(fn) - try: - with connections_filepath.open() as f: - self.connections_config = yamlutil.safe_load(f) - except EnvironmentError: - self.connections_config = { 'servers' : {} } - content = yamlutil.safe_dump(self.connections_config) - connections_filepath.setContent(content) - def init_stats_provider(self): gatherer_furl = self.get_config("client", "stats_gatherer.furl", None) self.stats_provider = StatsProvider(self, gatherer_furl) @@ -375,17 +359,28 @@ class Client(node.Node, pollmixin.PollMixin): self.storage_broker = sb sb.setServiceParent(self) - # utilize the loaded static server specifications - for key, server in self.connections_config['servers'].items(): - handlers = server.get("transport_handlers") - eventually(self.storage_broker.got_static_announcement, - key, server['announcement'], handlers) - sb.use_introducer(self.introducer_client) def get_storage_broker(self): return self.storage_broker + def load_static_servers(self): + """ + Load the servers.yaml file if it exists, and provide the static + server data to the StorageFarmBroker. + """ + fn = os.path.join(self.basedir, "private", "servers.yaml") + servers_filepath = FilePath(fn) + try: + with servers_filepath.open() as f: + servers_yaml = yamlutil.safe_load(f) + static_servers = servers_yaml.get("storage", {}) + log.msg("found %d static servers in private/servers.yaml" % + len(static_servers)) + self.storage_broker.set_static_servers(static_servers) + except EnvironmentError: + pass + def init_blacklist(self): fn = os.path.join(self.basedir, "access.blacklist") self.blacklist = Blacklist(fn) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a1285a099..1853a5596 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -80,11 +80,23 @@ class StorageFarmBroker(service.MultiService): # own Reconnector, and will give us a RemoteReference when we ask # them for it. self.servers = {} - self.static_servers = [] + self._static_server_ids = set() # ignore announcements for these self.introducer_client = None self._threshold_listeners = [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 + def set_static_servers(self, servers): + for (server_id, server) in servers.items(): + self._static_server_ids.add(server_id) + handlers = self._tub_handlers.copy() + handlers.update(server.get("connections", {})) + s = NativeStorageServer(server_id, server["ann"], + self._tub_options, handlers) + s.on_status_changed(lambda _: self._got_connection()) + s.setServiceParent(self) + self.servers[server_id] = s + s.start_connecting(self._trigger_connections) + def when_connected_enough(self, threshold): """ :returns: a Deferred that fires if/when our high water mark for @@ -103,9 +115,9 @@ class StorageFarmBroker(service.MultiService): s._is_connected = True self.servers[serverid] = s - def test_add_server(self, serverid, s): + def test_add_server(self, server_id, s): s.on_status_changed(lambda _: self._got_connection()) - self.servers[serverid] = s + self.servers[server_id] = s def use_introducer(self, introducer_client): self.introducer_client = ic = introducer_client @@ -128,24 +140,23 @@ class StorageFarmBroker(service.MultiService): remaining.append( (threshold, d) ) self._threshold_listeners = remaining - def got_static_announcement(self, key_s, ann, handlers): - server_id = key_s - assert server_id not in self.static_servers # XXX - self.static_servers.append(server_id) - self._got_announcement(key_s, ann, handlers=handlers) - - def _got_announcement(self, key_s, ann, handlers=None): + def _got_announcement(self, key_s, ann): precondition(isinstance(key_s, str), key_s) precondition(key_s.startswith("v0-"), key_s) precondition(ann["service-name"] == "storage", ann["service-name"]) - if handlers is not None: - s = NativeStorageServer(key_s, ann, self._tub_options, handlers) - else: - s = NativeStorageServer(key_s, ann, self._tub_options, self._tub_handlers) + server_id = key_s + if server_id in self._static_server_ids: + log.msg(format="ignoring announcement for static server '%(id)s'", + id=server_id, + facility="tahoe.storage_broker", umid="AlxzqA", + level=log.UNUSUAL) + return + s = NativeStorageServer(server_id, ann, + self._tub_options, self._tub_handlers) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) - if old and server_id not in self.static_servers: + if old: if old.get_announcement() == ann: return # duplicate # replacement @@ -265,9 +276,9 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, key_s, ann, tub_options={}, tub_handlers={}): + def __init__(self, server_id, ann, tub_options={}, tub_handlers={}): service.MultiService.__init__(self) - self.key_s = key_s + self._server_id = server_id self.announcement = ann self._tub_options = tub_options self._tub_handlers = tub_handlers @@ -282,16 +293,13 @@ class NativeStorageServer(service.MultiService): ps = base32.a2b(str(ann["permutation-seed-base32"])) self._permutation_seed = ps - if key_s: - self._long_description = key_s - if key_s.startswith("v0-"): - # remove v0- prefix from abbreviated name - self._short_description = key_s[3:3+8] - else: - self._short_description = key_s[:8] + assert server_id + self._long_description = server_id + if server_id.startswith("v0-"): + # remove v0- prefix from abbreviated name + self._short_description = server_id[3:3+8] else: - self._long_description = tubid_s - self._short_description = tubid_s[:6] + self._short_description = server_id[:8] self.last_connect_time = None self.last_loss_time = None @@ -321,7 +329,7 @@ class NativeStorageServer(service.MultiService): def __repr__(self): return "" % self.get_name() def get_serverid(self): - return self.key_s + return self._server_id def get_permutation_seed(self): return self._permutation_seed def get_version(self): @@ -340,7 +348,7 @@ class NativeStorageServer(service.MultiService): return self._tubid def get_nickname(self): - return self.announcement["nickname"] + return self.announcement.get("nickname", "") def get_announcement(self): return self.announcement def get_remote_host(self): diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index e39e4b095..475974c05 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -25,12 +25,12 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): sb = StorageFarmBroker(True) # s.get_name() (the "short description") will be "v0-00000000". # s.get_longname() will include the -long suffix. - # s.get_peerid() (i.e. tubid) will be "aaa.." or "777.." or "ceir.." servers = [("v0-00000000-long", "\x00"*20, "peer-0"), ("v0-ffffffff-long", "\xff"*20, "peer-f"), ("v0-11111111-long", "\x11"*20, "peer-11")] - for (key_s, peerid, nickname) in servers: - tubid_b32 = base32.b2a(peerid) + for (key_s, binary_tubid, nickname) in servers: + server_id = key_s + tubid_b32 = base32.b2a(binary_tubid) furl = "pb://%s@nowhere/fake" % tubid_b32 ann = { "version": 0, "service-name": "storage", @@ -41,8 +41,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(key_s, ann) - sb.test_add_server(peerid, s) # XXX: maybe use key_s? + s = NativeStorageServer(server_id, ann) + sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb return c diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 21eb3ac1c..2e7f63635 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -35,10 +35,16 @@ class TestNativeStorageServer(unittest.TestCase): }) self.failUnlessEqual(nss.get_available_space(), 111) + def test_missing_nickname(self): + ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + nss = NativeStorageServer("server_id", ann) + self.assertEqual(nss.get_nickname(), "") class TestStorageFarmBroker(unittest.TestCase): - def test_static_announcement(self): + def test_static_servers(self): broker = StorageFarmBroker(True) key_s = 'v0-1234-{}'.format(1) @@ -47,10 +53,26 @@ class TestStorageFarmBroker(unittest.TestCase): "anonymous-storage-FURL": "pb://{}@nowhere/fake".format(base32.b2a(str(1))), "permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa", } - broker.got_static_announcement(key_s, ann, None) - self.failUnlessEqual(len(broker.static_servers), 1) - self.failUnlessEqual(broker.servers[key_s].announcement, ann) - self.failUnlessEqual(broker.servers[key_s].key_s, key_s) + permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa") + broker.set_static_servers({key_s: {"ann": ann}}) + self.failUnlessEqual(len(broker._static_server_ids), 1) + s = broker.servers[key_s] + self.failUnlessEqual(s.announcement, ann) + self.failUnlessEqual(s.get_serverid(), key_s) + self.assertEqual(s.get_permutation_seed(), permseed) + + # if the Introducer announces the same thing, we're supposed to + # ignore it + + ann2 = { + "service-name": "storage", + "anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(str(1))), + "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", + } + broker._got_announcement(key_s, ann2) + s2 = broker.servers[key_s] + self.assertIdentical(s2, s) + self.assertEqual(s2.get_permutation_seed(), permseed) @inlineCallbacks def test_threshold_reached(self): diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py new file mode 100644 index 000000000..c9e5784c1 --- /dev/null +++ b/src/allmydata/test/web/test_root.py @@ -0,0 +1,35 @@ +from twisted.trial import unittest + +from ...storage_client import NativeStorageServer +from ...web.root import Root + +class FakeRoot(Root): + def __init__(self): + pass + def now_fn(self): + return 0 + +class FakeContext: + def __init__(self): + self.slots = {} + self.tag = self + def fillSlots(self, slotname, contents): + self.slots[slotname] = contents + +class RenderServiceRow(unittest.TestCase): + def test_missing(self): + # minimally-defined static servers just need anonymous-storage-FURL + # and permutation-seed-base32. The WUI used to have problems + # rendering servers that lacked nickname and version. This tests that + # we can render such minimal servers. + ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + s = NativeStorageServer("server_id", ann) + + r = FakeRoot() + ctx = FakeContext() + res = r.render_service_row(ctx, s) + self.assertIdentical(res, ctx) + self.assertEqual(ctx.slots["version"], "") + self.assertEqual(ctx.slots["nickname"], "") diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 8b5f196a3..205d3dcac 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -329,7 +329,7 @@ class Root(rend.Page): last_received_data_abs_time = render_time_attr(last_received_data_time) announcement = server.get_announcement() - version = announcement["my-version"] + version = announcement.get("my-version", "") available_space = server.get_available_space() if available_space is None: available_space = "N/A"