From f23660e1782864f2447a1a818456761f4e6cef87 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 26 Aug 2016 17:29:39 -0700 Subject: [PATCH 1/4] NativeStorageServer: create with server_id, not key_s They're the same thing, but knowing that is the responsibility of the caller, not NativeStorageServer. Try to normalize on "server_id" as the spelling. Remove support for missing key_s, now that we require V2 introductions. --- src/allmydata/storage_client.py | 25 ++++++++++------------- src/allmydata/test/test_checker.py | 10 ++++----- src/allmydata/test/test_storage_client.py | 2 +- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index a1285a099..b44039e45 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -103,9 +103,9 @@ class StorageFarmBroker(service.MultiService): s._is_connected = True self.servers[serverid] = s - def test_add_server(self, serverid, s): + def test_add_server(self, server_id, s): s.on_status_changed(lambda _: self._got_connection()) - self.servers[serverid] = s + self.servers[server_id] = s def use_introducer(self, introducer_client): self.introducer_client = ic = introducer_client @@ -265,9 +265,9 @@ class NativeStorageServer(service.MultiService): "application-version": "unknown: no get_version()", } - def __init__(self, key_s, ann, tub_options={}, tub_handlers={}): + def __init__(self, server_id, ann, tub_options={}, tub_handlers={}): service.MultiService.__init__(self) - self.key_s = key_s + self._server_id = server_id self.announcement = ann self._tub_options = tub_options self._tub_handlers = tub_handlers @@ -282,16 +282,13 @@ class NativeStorageServer(service.MultiService): ps = base32.a2b(str(ann["permutation-seed-base32"])) self._permutation_seed = ps - if key_s: - self._long_description = key_s - if key_s.startswith("v0-"): - # remove v0- prefix from abbreviated name - self._short_description = key_s[3:3+8] - else: - self._short_description = key_s[:8] + assert server_id + self._long_description = server_id + if server_id.startswith("v0-"): + # remove v0- prefix from abbreviated name + self._short_description = server_id[3:3+8] else: - self._long_description = tubid_s - self._short_description = tubid_s[:6] + self._short_description = server_id[:8] self.last_connect_time = None self.last_loss_time = None @@ -321,7 +318,7 @@ class NativeStorageServer(service.MultiService): def __repr__(self): return "" % self.get_name() def get_serverid(self): - return self.key_s + return self._server_id def get_permutation_seed(self): return self._permutation_seed def get_version(self): diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index e39e4b095..475974c05 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -25,12 +25,12 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): sb = StorageFarmBroker(True) # s.get_name() (the "short description") will be "v0-00000000". # s.get_longname() will include the -long suffix. - # s.get_peerid() (i.e. tubid) will be "aaa.." or "777.." or "ceir.." servers = [("v0-00000000-long", "\x00"*20, "peer-0"), ("v0-ffffffff-long", "\xff"*20, "peer-f"), ("v0-11111111-long", "\x11"*20, "peer-11")] - for (key_s, peerid, nickname) in servers: - tubid_b32 = base32.b2a(peerid) + for (key_s, binary_tubid, nickname) in servers: + server_id = key_s + tubid_b32 = base32.b2a(binary_tubid) furl = "pb://%s@nowhere/fake" % tubid_b32 ann = { "version": 0, "service-name": "storage", @@ -41,8 +41,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin): "my-version": "ver", "oldest-supported": "oldest", } - s = NativeStorageServer(key_s, ann) - sb.test_add_server(peerid, s) # XXX: maybe use key_s? + s = NativeStorageServer(server_id, ann) + sb.test_add_server(server_id, s) c = FakeClient() c.storage_broker = sb return c diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 21eb3ac1c..92909f81c 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -50,7 +50,7 @@ class TestStorageFarmBroker(unittest.TestCase): broker.got_static_announcement(key_s, ann, None) self.failUnlessEqual(len(broker.static_servers), 1) self.failUnlessEqual(broker.servers[key_s].announcement, ann) - self.failUnlessEqual(broker.servers[key_s].key_s, key_s) + self.failUnlessEqual(broker.servers[key_s].get_serverid(), key_s) @inlineCallbacks def test_threshold_reached(self): From d75b9f822a272138d63e1feac51e203709e30df4 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 26 Aug 2016 17:31:02 -0700 Subject: [PATCH 2/4] Improve loading of static servers This follows the latest comments in ticket:2788, moving the static server definitions from "connections.yaml" to "servers.yaml". It removes the "connections" and "introducers" blocks from that file, leaving it responsible for just static servers (I think connections and introducers can be configured from tahoe.cfg). This feeds all the static server specs to the StorageFarmBroker in a single call, rather than delivering them as simulated introducer announcements. It cleans up the way handlers are specified too (the handler dictionary is ignored, but that will change soon). --- src/allmydata/client.py | 41 ++++++++++------------- src/allmydata/storage_client.py | 37 +++++++++++++------- src/allmydata/test/test_storage_client.py | 26 +++++++++++--- 3 files changed, 63 insertions(+), 41 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index d21e9faee..b96bcffb4 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -8,7 +8,6 @@ from twisted.application import service from twisted.application.internet import TimerService from twisted.python.filepath import FilePath from pycryptopp.publickey import rsa -from foolscap.api import eventually import allmydata from allmydata.storage.server import StorageServer @@ -128,7 +127,6 @@ class Client(node.Node, pollmixin.PollMixin): self.started_timestamp = time.time() self.logSource="Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() - self.load_connections() self.init_introducer_client() self.init_stats_provider() self.init_secrets() @@ -140,6 +138,7 @@ class Client(node.Node, pollmixin.PollMixin): if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") self.init_client() + self.load_static_servers() self.helper = None if self.get_config("helper", "enabled", False, boolean=True): self.init_helper() @@ -186,21 +185,6 @@ class Client(node.Node, pollmixin.PollMixin): self.introducer_client = ic ic.setServiceParent(self) - def load_connections(self): - """ - Load the connections.yaml file if it exists, otherwise - create a default configuration. - """ - fn = os.path.join(self.basedir, "private", "connections.yaml") - connections_filepath = FilePath(fn) - try: - with connections_filepath.open() as f: - self.connections_config = yamlutil.safe_load(f) - except EnvironmentError: - self.connections_config = { 'servers' : {} } - content = yamlutil.safe_dump(self.connections_config) - connections_filepath.setContent(content) - def init_stats_provider(self): gatherer_furl = self.get_config("client", "stats_gatherer.furl", None) self.stats_provider = StatsProvider(self, gatherer_furl) @@ -375,17 +359,28 @@ class Client(node.Node, pollmixin.PollMixin): self.storage_broker = sb sb.setServiceParent(self) - # utilize the loaded static server specifications - for key, server in self.connections_config['servers'].items(): - handlers = server.get("transport_handlers") - eventually(self.storage_broker.got_static_announcement, - key, server['announcement'], handlers) - sb.use_introducer(self.introducer_client) def get_storage_broker(self): return self.storage_broker + def load_static_servers(self): + """ + Load the servers.yaml file if it exists, and provide the static + server data to the StorageFarmBroker. + """ + fn = os.path.join(self.basedir, "private", "servers.yaml") + servers_filepath = FilePath(fn) + try: + with servers_filepath.open() as f: + servers_yaml = yamlutil.safe_load(f) + static_servers = servers_yaml.get("storage", {}) + log.msg("found %d static servers in private/servers.yaml" % + len(static_servers)) + self.storage_broker.set_static_servers(static_servers) + except EnvironmentError: + pass + def init_blacklist(self): fn = os.path.join(self.basedir, "access.blacklist") self.blacklist = Blacklist(fn) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index b44039e45..3217b5618 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -80,11 +80,23 @@ class StorageFarmBroker(service.MultiService): # own Reconnector, and will give us a RemoteReference when we ask # them for it. self.servers = {} - self.static_servers = [] + self._static_server_ids = set() # ignore announcements for these self.introducer_client = None self._threshold_listeners = [] # tuples of (threshold, Deferred) self._connected_high_water_mark = 0 + def set_static_servers(self, servers): + for (server_id, server) in servers.items(): + self._static_server_ids.add(server_id) + handlers = self._tub_handlers.copy() + handlers.update(server.get("connections", {})) + s = NativeStorageServer(server_id, server["ann"], + self._tub_options, handlers) + s.on_status_changed(lambda _: self._got_connection()) + s.setServiceParent(self) + self.servers[server_id] = s + s.start_connecting(self._trigger_connections) + def when_connected_enough(self, threshold): """ :returns: a Deferred that fires if/when our high water mark for @@ -128,24 +140,23 @@ class StorageFarmBroker(service.MultiService): remaining.append( (threshold, d) ) self._threshold_listeners = remaining - def got_static_announcement(self, key_s, ann, handlers): - server_id = key_s - assert server_id not in self.static_servers # XXX - self.static_servers.append(server_id) - self._got_announcement(key_s, ann, handlers=handlers) - - def _got_announcement(self, key_s, ann, handlers=None): + def _got_announcement(self, key_s, ann): precondition(isinstance(key_s, str), key_s) precondition(key_s.startswith("v0-"), key_s) precondition(ann["service-name"] == "storage", ann["service-name"]) - if handlers is not None: - s = NativeStorageServer(key_s, ann, self._tub_options, handlers) - else: - s = NativeStorageServer(key_s, ann, self._tub_options, self._tub_handlers) + server_id = key_s + if server_id in self._static_server_ids: + log.msg(format="ignoring announcement for static server '%(id)s'", + id=server_id, + facility="tahoe.storage_broker", umid="AlxzqA", + level=log.UNUSUAL) + return + s = NativeStorageServer(server_id, ann, + self._tub_options, self._tub_handlers) s.on_status_changed(lambda _: self._got_connection()) server_id = s.get_serverid() old = self.servers.get(server_id) - if old and server_id not in self.static_servers: + if old: if old.get_announcement() == ann: return # duplicate # replacement diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 92909f81c..e1c59a8f3 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -38,7 +38,7 @@ class TestNativeStorageServer(unittest.TestCase): class TestStorageFarmBroker(unittest.TestCase): - def test_static_announcement(self): + def test_static_servers(self): broker = StorageFarmBroker(True) key_s = 'v0-1234-{}'.format(1) @@ -47,10 +47,26 @@ class TestStorageFarmBroker(unittest.TestCase): "anonymous-storage-FURL": "pb://{}@nowhere/fake".format(base32.b2a(str(1))), "permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa", } - broker.got_static_announcement(key_s, ann, None) - self.failUnlessEqual(len(broker.static_servers), 1) - self.failUnlessEqual(broker.servers[key_s].announcement, ann) - self.failUnlessEqual(broker.servers[key_s].get_serverid(), key_s) + permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa") + broker.set_static_servers({key_s: {"ann": ann}}) + self.failUnlessEqual(len(broker._static_server_ids), 1) + s = broker.servers[key_s] + self.failUnlessEqual(s.announcement, ann) + self.failUnlessEqual(s.get_serverid(), key_s) + self.assertEqual(s.get_permutation_seed(), permseed) + + # if the Introducer announces the same thing, we're supposed to + # ignore it + + ann2 = { + "service-name": "storage", + "anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(str(1))), + "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", + } + broker._got_announcement(key_s, ann2) + s2 = broker.servers[key_s] + self.assertIdentical(s2, s) + self.assertEqual(s2.get_permutation_seed(), permseed) @inlineCallbacks def test_threshold_reached(self): From 859ce66a0318b608d2f6d110fdb285c930a0ce86 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 26 Aug 2016 18:03:53 -0700 Subject: [PATCH 3/4] document private/servers.yaml (static servers) --- docs/configuration.rst | 55 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 2e9f216b1..92ca7e935 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -12,8 +12,9 @@ Configuring a Tahoe-LAFS node 6. `Running A Helper`_ 7. `Running An Introducer`_ 8. `Other Files in BASEDIR`_ -9. `Other files`_ -10. `Example`_ +9. `Static Server Definitions`_ +10. `Other files`_ +11. `Example`_ A Tahoe-LAFS node is configured by writing to files in its base directory. These files are read by the node when it starts, so each time you change @@ -662,6 +663,56 @@ This section describes these other files. ``private/convergence`` is a zero-length file). +Static Server Definitions +========================= + +The ``private/servers.yaml`` file defines "static servers": those which are +not announced through the Introducer. This can also control how we connect to +those servers. + +Most clients do not need this file. It is only necessary if you want to use +servers which are (for some specialized reason) not announced through the +Introducer, or to connect to those servers in different ways. You might do +this to "freeze" the server list: use the Introducer for a while, then copy +all announcements into ``servers.yaml``, then stop using the Introducer +entirely. Or you might have a private server that you don't want other users +to learn about (via the Introducer). Or you might run a local server which is +announced to everyone else as a Tor onion address, but which you can connect +to directly (via TCP). + +The file syntax is `YAML`_, with a top-level dictionary named ``storage``. +Other items may be added in the future. + +The ``storage`` dictionary takes keys which are server-ids, and values which +are dictionaries with two keys: ``ann`` and ``connections``. The ``ann`` +value is a dictionary which will be used in lieu of the introducer +announcement, so it can be populated by copying the ``ann`` dictionary from +``NODEDIR/introducer_cache.yaml``. Static servers which use the node's +default connection handlers only need a few keys: + +* the server ID, which can be any string +* a nickname, which is the string that is printed on the web interface +* the ``anonymous-storage-FURL``, which is where the server lives +* ``permutation-seed-base32``, which controls how shares are mapped to + servers. This is normally computed from the server-ID, but can be + overridden to maintain the mapping for older servers which used to use + Foolscap TubIDs as server-IDs. +* more important keys may be added in the future, as Accounting and + HTTP-based servers are implemented + +For example, a private static server could be defined with a +``private/servers.yaml`` file like this:: + + storage: + my-serverid-1: + ann: + nickname: my-server-1 + anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:8.8.8.8:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b + permutation-seed-base32: w2hqnbaa25yw4qgcvghl5psa3srpfgw3 + +.. _YAML: http://yaml.org/ + + Other files =========== From 663e39593bbd7aa1829c208ae8c1aed5ca0e18e2 Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Fri, 26 Aug 2016 17:55:52 -0700 Subject: [PATCH 4/4] static servers: tolerate missing nickname/versions A minimally-defined static server only specifies server_id, anonymous-storage-FURL, and permutation-seed-base32. But the WUI Welcome page wouldn't render (it raised an exception) without also defining nickname and version. This allows those values to be missing. --- src/allmydata/storage_client.py | 2 +- src/allmydata/test/test_storage_client.py | 6 ++++ src/allmydata/test/web/test_root.py | 35 +++++++++++++++++++++++ src/allmydata/web/root.py | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 src/allmydata/test/web/test_root.py diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 3217b5618..1853a5596 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -348,7 +348,7 @@ class NativeStorageServer(service.MultiService): return self._tubid def get_nickname(self): - return self.announcement["nickname"] + return self.announcement.get("nickname", "") def get_announcement(self): return self.announcement def get_remote_host(self): diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index e1c59a8f3..2e7f63635 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -35,6 +35,12 @@ class TestNativeStorageServer(unittest.TestCase): }) self.failUnlessEqual(nss.get_available_space(), 111) + def test_missing_nickname(self): + ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + nss = NativeStorageServer("server_id", ann) + self.assertEqual(nss.get_nickname(), "") class TestStorageFarmBroker(unittest.TestCase): diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py new file mode 100644 index 000000000..c9e5784c1 --- /dev/null +++ b/src/allmydata/test/web/test_root.py @@ -0,0 +1,35 @@ +from twisted.trial import unittest + +from ...storage_client import NativeStorageServer +from ...web.root import Root + +class FakeRoot(Root): + def __init__(self): + pass + def now_fn(self): + return 0 + +class FakeContext: + def __init__(self): + self.slots = {} + self.tag = self + def fillSlots(self, slotname, contents): + self.slots[slotname] = contents + +class RenderServiceRow(unittest.TestCase): + def test_missing(self): + # minimally-defined static servers just need anonymous-storage-FURL + # and permutation-seed-base32. The WUI used to have problems + # rendering servers that lacked nickname and version. This tests that + # we can render such minimal servers. + ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", + "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", + } + s = NativeStorageServer("server_id", ann) + + r = FakeRoot() + ctx = FakeContext() + res = r.render_service_row(ctx, s) + self.assertIdentical(res, ctx) + self.assertEqual(ctx.slots["version"], "") + self.assertEqual(ctx.slots["nickname"], "") diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 8b5f196a3..205d3dcac 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -329,7 +329,7 @@ class Root(rend.Page): last_received_data_abs_time = render_time_attr(last_received_data_time) announcement = server.get_announcement() - version = announcement["my-version"] + version = announcement.get("my-version", "") available_space = server.get_available_space() if available_space is None: available_space = "N/A"