Merge 2788-load-static-servers: add servers.yaml

Closes tahoe-lafs/tahoe-lafs#319 (in rebased form, with some additional
tests and better docs)

refs ticket:2788

docs/configuration.rst

@@ -12,8 +12,9 @@ Configuring a Tahoe-LAFS node
 6.  `Running A Helper`_
 7.  `Running An Introducer`_
 8.  `Other Files in BASEDIR`_
-9.  `Other files`_
-10. `Example`_
+9.  `Static Server Definitions`_
+10. `Other files`_
+11. `Example`_
 
 A Tahoe-LAFS node is configured by writing to files in its base directory.
 These files are read by the node when it starts, so each time you change
@@ -662,6 +663,56 @@ This section describes these other files.
   ``private/convergence`` is a zero-length file).
 
 
+Static Server Definitions
+=========================
+
+The ``private/servers.yaml`` file defines "static servers": those which are
+not announced through the Introducer. This can also control how we connect to
+those servers.
+
+Most clients do not need this file. It is only necessary if you want to use
+servers which are (for some specialized reason) not announced through the
+Introducer, or to connect to those servers in different ways. You might do
+this to "freeze" the server list: use the Introducer for a while, then copy
+all announcements into ``servers.yaml``, then stop using the Introducer
+entirely. Or you might have a private server that you don't want other users
+to learn about (via the Introducer). Or you might run a local server which is
+announced to everyone else as a Tor onion address, but which you can connect
+to directly (via TCP).
+
+The file syntax is `YAML`_, with a top-level dictionary named ``storage``.
+Other items may be added in the future.
+
+The ``storage`` dictionary takes keys which are server-ids, and values which
+are dictionaries with two keys: ``ann`` and ``connections``. The ``ann``
+value is a dictionary which will be used in lieu of the introducer
+announcement, so it can be populated by copying the ``ann`` dictionary from
+``NODEDIR/introducer_cache.yaml``. Static servers which use the node's
+default connection handlers only need a few keys:
+
+* the server ID, which can be any string
+* a nickname, which is the string that is printed on the web interface
+* the ``anonymous-storage-FURL``, which is where the server lives
+* ``permutation-seed-base32``, which controls how shares are mapped to
+  servers. This is normally computed from the server-ID, but can be
+  overridden to maintain the mapping for older servers which used to use
+  Foolscap TubIDs as server-IDs.
+* more important keys may be added in the future, as Accounting and
+  HTTP-based servers are implemented
+
+For example, a private static server could be defined with a
+``private/servers.yaml`` file like this::
+
+  storage:
+    my-serverid-1:
+      ann:
+        nickname: my-server-1
+        anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:8.8.8.8:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b
+        permutation-seed-base32: w2hqnbaa25yw4qgcvghl5psa3srpfgw3
+
+.. _YAML: http://yaml.org/
+
+
 Other files
 ===========
 

src/allmydata/client.py

@@ -8,7 +8,6 @@ from twisted.application import service
 from twisted.application.internet import TimerService
 from twisted.python.filepath import FilePath
 from pycryptopp.publickey import rsa
-from foolscap.api import eventually
 
 import allmydata
 from allmydata.storage.server import StorageServer
@@ -128,7 +127,6 @@ class Client(node.Node, pollmixin.PollMixin):
         self.started_timestamp = time.time()
         self.logSource="Client"
         self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
-        self.load_connections()
         self.init_introducer_client()
         self.init_stats_provider()
         self.init_secrets()
@@ -140,6 +138,7 @@ class Client(node.Node, pollmixin.PollMixin):
         if key_gen_furl:
             log.msg("[client]key_generator.furl= is now ignored, see #2783")
         self.init_client()
+        self.load_static_servers()
         self.helper = None
         if self.get_config("helper", "enabled", False, boolean=True):
             self.init_helper()
@@ -186,21 +185,6 @@ class Client(node.Node, pollmixin.PollMixin):
         self.introducer_client = ic
         ic.setServiceParent(self)
 
-    def load_connections(self):
-        """
-        Load the connections.yaml file if it exists, otherwise
-        create a default configuration.
-        """
-        fn = os.path.join(self.basedir, "private", "connections.yaml")
-        connections_filepath = FilePath(fn)
-        try:
-            with connections_filepath.open() as f:
-                self.connections_config = yamlutil.safe_load(f)
-        except EnvironmentError:
-            self.connections_config = { 'servers' : {} }
-            content = yamlutil.safe_dump(self.connections_config)
-            connections_filepath.setContent(content)
-
     def init_stats_provider(self):
         gatherer_furl = self.get_config("client", "stats_gatherer.furl", None)
         self.stats_provider = StatsProvider(self, gatherer_furl)
@@ -375,17 +359,28 @@ class Client(node.Node, pollmixin.PollMixin):
         self.storage_broker = sb
         sb.setServiceParent(self)
 
-        # utilize the loaded static server specifications
-        for key, server in self.connections_config['servers'].items():
-            handlers = server.get("transport_handlers")
-            eventually(self.storage_broker.got_static_announcement,
-                       key, server['announcement'], handlers)
-
         sb.use_introducer(self.introducer_client)
 
     def get_storage_broker(self):
         return self.storage_broker
 
+    def load_static_servers(self):
+        """
+        Load the servers.yaml file if it exists, and provide the static
+        server data to the StorageFarmBroker.
+        """
+        fn = os.path.join(self.basedir, "private", "servers.yaml")
+        servers_filepath = FilePath(fn)
+        try:
+            with servers_filepath.open() as f:
+                servers_yaml = yamlutil.safe_load(f)
+            static_servers = servers_yaml.get("storage", {})
+            log.msg("found %d static servers in private/servers.yaml" %
+                    len(static_servers))
+            self.storage_broker.set_static_servers(static_servers)
+        except EnvironmentError:
+            pass
+
     def init_blacklist(self):
         fn = os.path.join(self.basedir, "access.blacklist")
         self.blacklist = Blacklist(fn)

src/allmydata/storage_client.py

@@ -80,11 +80,23 @@ class StorageFarmBroker(service.MultiService):
         # own Reconnector, and will give us a RemoteReference when we ask
         # them for it.
         self.servers = {}
-        self.static_servers = []
+        self._static_server_ids = set() # ignore announcements for these
         self.introducer_client = None
         self._threshold_listeners = [] # tuples of (threshold, Deferred)
         self._connected_high_water_mark = 0
 
+    def set_static_servers(self, servers):
+        for (server_id, server) in servers.items():
+            self._static_server_ids.add(server_id)
+            handlers = self._tub_handlers.copy()
+            handlers.update(server.get("connections", {}))
+            s = NativeStorageServer(server_id, server["ann"],
+                                    self._tub_options, handlers)
+            s.on_status_changed(lambda _: self._got_connection())
+            s.setServiceParent(self)
+            self.servers[server_id] = s
+            s.start_connecting(self._trigger_connections)
+
     def when_connected_enough(self, threshold):
         """
         :returns: a Deferred that fires if/when our high water mark for
@@ -103,9 +115,9 @@ class StorageFarmBroker(service.MultiService):
         s._is_connected = True
         self.servers[serverid] = s
 
-    def test_add_server(self, serverid, s):
+    def test_add_server(self, server_id, s):
         s.on_status_changed(lambda _: self._got_connection())
-        self.servers[serverid] = s
+        self.servers[server_id] = s
 
     def use_introducer(self, introducer_client):
         self.introducer_client = ic = introducer_client
@@ -128,24 +140,23 @@ class StorageFarmBroker(service.MultiService):
                 remaining.append( (threshold, d) )
         self._threshold_listeners = remaining
 
-    def got_static_announcement(self, key_s, ann, handlers):
-        server_id = key_s
-        assert server_id not in self.static_servers # XXX
-        self.static_servers.append(server_id)
-        self._got_announcement(key_s, ann, handlers=handlers)
-
-    def _got_announcement(self, key_s, ann, handlers=None):
+    def _got_announcement(self, key_s, ann):
         precondition(isinstance(key_s, str), key_s)
         precondition(key_s.startswith("v0-"), key_s)
         precondition(ann["service-name"] == "storage", ann["service-name"])
-        if handlers is not None:
-            s = NativeStorageServer(key_s, ann, self._tub_options, handlers)
-        else:
-            s = NativeStorageServer(key_s, ann, self._tub_options, self._tub_handlers)
+        server_id = key_s
+        if server_id in self._static_server_ids:
+            log.msg(format="ignoring announcement for static server '%(id)s'",
+                    id=server_id,
+                    facility="tahoe.storage_broker", umid="AlxzqA",
+                    level=log.UNUSUAL)
+            return
+        s = NativeStorageServer(server_id, ann,
+                                self._tub_options, self._tub_handlers)
         s.on_status_changed(lambda _: self._got_connection())
         server_id = s.get_serverid()
         old = self.servers.get(server_id)
-        if old and server_id not in self.static_servers:
+        if old:
             if old.get_announcement() == ann:
                 return # duplicate
             # replacement
@@ -265,9 +276,9 @@ class NativeStorageServer(service.MultiService):
         "application-version": "unknown: no get_version()",
         }
 
-    def __init__(self, key_s, ann, tub_options={}, tub_handlers={}):
+    def __init__(self, server_id, ann, tub_options={}, tub_handlers={}):
         service.MultiService.__init__(self)
-        self.key_s = key_s
+        self._server_id = server_id
         self.announcement = ann
         self._tub_options = tub_options
         self._tub_handlers = tub_handlers
@@ -282,16 +293,13 @@ class NativeStorageServer(service.MultiService):
         ps = base32.a2b(str(ann["permutation-seed-base32"]))
         self._permutation_seed = ps
 
-        if key_s:
-            self._long_description = key_s
-            if key_s.startswith("v0-"):
-                # remove v0- prefix from abbreviated name
-                self._short_description = key_s[3:3+8]
-            else:
-                self._short_description = key_s[:8]
+        assert server_id
+        self._long_description = server_id
+        if server_id.startswith("v0-"):
+            # remove v0- prefix from abbreviated name
+            self._short_description = server_id[3:3+8]
         else:
-            self._long_description = tubid_s
-            self._short_description = tubid_s[:6]
+            self._short_description = server_id[:8]
 
         self.last_connect_time = None
         self.last_loss_time = None
@@ -321,7 +329,7 @@ class NativeStorageServer(service.MultiService):
     def __repr__(self):
         return "<NativeStorageServer for %s>" % self.get_name()
     def get_serverid(self):
-        return self.key_s
+        return self._server_id
     def get_permutation_seed(self):
         return self._permutation_seed
     def get_version(self):
@@ -340,7 +348,7 @@ class NativeStorageServer(service.MultiService):
         return self._tubid
 
     def get_nickname(self):
-        return self.announcement["nickname"]
+        return self.announcement.get("nickname", "")
     def get_announcement(self):
         return self.announcement
     def get_remote_host(self):

src/allmydata/test/test_checker.py

@@ -25,12 +25,12 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
         sb = StorageFarmBroker(True)
         # s.get_name() (the "short description") will be "v0-00000000".
         # s.get_longname() will include the -long suffix.
-        # s.get_peerid() (i.e. tubid) will be "aaa.." or "777.." or "ceir.."
         servers = [("v0-00000000-long", "\x00"*20, "peer-0"),
                    ("v0-ffffffff-long", "\xff"*20, "peer-f"),
                    ("v0-11111111-long", "\x11"*20, "peer-11")]
-        for (key_s, peerid, nickname) in servers:
-            tubid_b32 = base32.b2a(peerid)
+        for (key_s, binary_tubid, nickname) in servers:
+            server_id = key_s
+            tubid_b32 = base32.b2a(binary_tubid)
             furl = "pb://%s@nowhere/fake" % tubid_b32
             ann = { "version": 0,
                     "service-name": "storage",
@@ -41,8 +41,8 @@ class WebResultsRendering(unittest.TestCase, WebRenderingMixin):
                     "my-version": "ver",
                     "oldest-supported": "oldest",
                     }
-            s = NativeStorageServer(key_s, ann)
-            sb.test_add_server(peerid, s) # XXX: maybe use key_s?
+            s = NativeStorageServer(server_id, ann)
+            sb.test_add_server(server_id, s)
         c = FakeClient()
         c.storage_broker = sb
         return c

src/allmydata/test/test_storage_client.py

@@ -35,10 +35,16 @@ class TestNativeStorageServer(unittest.TestCase):
             })
         self.failUnlessEqual(nss.get_available_space(), 111)
 
+    def test_missing_nickname(self):
+        ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
+               "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
+               }
+        nss = NativeStorageServer("server_id", ann)
+        self.assertEqual(nss.get_nickname(), "")
 
 class TestStorageFarmBroker(unittest.TestCase):
 
-    def test_static_announcement(self):
+    def test_static_servers(self):
         broker = StorageFarmBroker(True)
 
         key_s = 'v0-1234-{}'.format(1)
@@ -47,10 +53,26 @@ class TestStorageFarmBroker(unittest.TestCase):
             "anonymous-storage-FURL": "pb://{}@nowhere/fake".format(base32.b2a(str(1))),
             "permutation-seed-base32": "aaaaaaaaaaaaaaaaaaaaaaaa",
         }
-        broker.got_static_announcement(key_s, ann, None)
-        self.failUnlessEqual(len(broker.static_servers), 1)
-        self.failUnlessEqual(broker.servers[key_s].announcement, ann)
-        self.failUnlessEqual(broker.servers[key_s].key_s, key_s)
+        permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa")
+        broker.set_static_servers({key_s: {"ann": ann}})
+        self.failUnlessEqual(len(broker._static_server_ids), 1)
+        s = broker.servers[key_s]
+        self.failUnlessEqual(s.announcement, ann)
+        self.failUnlessEqual(s.get_serverid(), key_s)
+        self.assertEqual(s.get_permutation_seed(), permseed)
+
+        # if the Introducer announces the same thing, we're supposed to
+        # ignore it
+
+        ann2 = {
+            "service-name": "storage",
+            "anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(str(1))),
+            "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
+        }
+        broker._got_announcement(key_s, ann2)
+        s2 = broker.servers[key_s]
+        self.assertIdentical(s2, s)
+        self.assertEqual(s2.get_permutation_seed(), permseed)
 
     @inlineCallbacks
     def test_threshold_reached(self):

src/allmydata/test/web/test_root.py

@@ -0,0 +1,35 @@
+from twisted.trial import unittest
+
+from ...storage_client import NativeStorageServer
+from ...web.root import Root
+
+class FakeRoot(Root):
+    def __init__(self):
+        pass
+    def now_fn(self):
+        return 0
+
+class FakeContext:
+    def __init__(self):
+        self.slots = {}
+        self.tag = self
+    def fillSlots(self, slotname, contents):
+        self.slots[slotname] = contents
+
+class RenderServiceRow(unittest.TestCase):
+    def test_missing(self):
+        # minimally-defined static servers just need anonymous-storage-FURL
+        # and permutation-seed-base32. The WUI used to have problems
+        # rendering servers that lacked nickname and version. This tests that
+        # we can render such minimal servers.
+        ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
+               "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
+               }
+        s = NativeStorageServer("server_id", ann)
+
+        r = FakeRoot()
+        ctx = FakeContext()
+        res = r.render_service_row(ctx, s)
+        self.assertIdentical(res, ctx)
+        self.assertEqual(ctx.slots["version"], "")
+        self.assertEqual(ctx.slots["nickname"], "")

src/allmydata/web/root.py

@@ -329,7 +329,7 @@ class Root(rend.Page):
         last_received_data_abs_time = render_time_attr(last_received_data_time)
 
         announcement = server.get_announcement()
-        version = announcement["my-version"]
+        version = announcement.get("my-version", "")
         available_space = server.get_available_space()
         if available_space is None:
             available_space = "N/A"