Add IntroducerClient write-only yaml cache file

this change also includes unit tests

src/allmydata/_auto_deps.py

@@ -87,6 +87,7 @@ install_requires = [
     # * pyOpenSSL >= 0.14 is needed in order to avoid
     #   <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2474>.
     "pyOpenSSL >= 0.14",
+    "PyYAML >= 3.11",
 ]
 
 # Includes some indirect dependencies, but does not include allmydata.
@@ -114,6 +115,7 @@ package_imports = [
     ('six',              'six'),
     ('enum34',           'enum'),
     ('pycparser',        'pycparser'),
+    ('PyYAML',           'yaml'),
 ]
 
 # Dependencies for which we don't know how to get a version number at run-time.

src/allmydata/client.py

@@ -6,6 +6,7 @@ from zope.interface import implements
 from twisted.internet import reactor, defer
 from twisted.application import service
 from twisted.application.internet import TimerService
+from twisted.python.filepath import FilePath
 from pycryptopp.publickey import rsa
 
 import allmydata
@@ -171,12 +172,13 @@ class Client(node.Node, pollmixin.PollMixin):
 
     def init_introducer_client(self):
         self.introducer_furl = self.get_config("client", "introducer.furl")
+        introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml"))
         ic = IntroducerClient(self.tub, self.introducer_furl,
                               self.nickname,
                               str(allmydata.__full_version__),
                               str(self.OLDEST_SUPPORTED_VERSION),
                               self.get_app_versions(),
-                              self._sequencer)
+                              self._sequencer, introducer_cache_filepath)
         self.introducer_client = ic
         ic.setServiceParent(self)
 

src/allmydata/introducer/client.py

@@ -1,5 +1,5 @@
 
-import time
+import time, yaml
 from zope.interface import implements
 from twisted.application import service
 from foolscap.api import Referenceable, eventually, RemoteInterface
@@ -47,7 +47,7 @@ class IntroducerClient(service.Service, Referenceable):
 
     def __init__(self, tub, introducer_furl,
                  nickname, my_version, oldest_supported,
-                 app_versions, sequencer):
+                 app_versions, sequencer, cache_filepath):
         self._tub = tub
         self.introducer_furl = introducer_furl
 
@@ -57,6 +57,7 @@ class IntroducerClient(service.Service, Referenceable):
         self._oldest_supported = oldest_supported
         self._app_versions = app_versions
         self._sequencer = sequencer
+        self._cache_filepath = cache_filepath
 
         self._my_subscriber_info = { "version": 0,
                                      "nickname": self._nickname,
@@ -112,6 +113,25 @@ class IntroducerClient(service.Service, Referenceable):
                      level=log.WEIRD, failure=failure, umid="c5MqUQ")
         d = self._tub.getReference(self.introducer_furl)
         d.addErrback(connect_failed)
+        def remove_cache(result):
+            try:
+                self._cache_filepath.remove()
+            except OSError, e:
+                pass
+            return result
+        d.addCallback(remove_cache)
+
+    def _save_announcements(self):
+        announcements = []
+        for _, value in self._inbound_announcements.items():
+            ann, key_s, time_stamp = value
+            server_params = {
+                "ann" : ann,
+                "key_s" : key_s,
+                }
+            announcements.append(server_params)
+        announcement_cache_yaml = yaml.dump(announcements)
+        self._cache_filepath.setContent(announcement_cache_yaml)
 
     def _got_introducer(self, publisher):
         self.log("connected to introducer, getting versions")
@@ -148,6 +168,7 @@ class IntroducerClient(service.Service, Referenceable):
         return log.msg(*args, **kwargs)
 
     def subscribe_to(self, service_name, cb, *args, **kwargs):
+        self._got_announcement_cb = cb
         self._local_subscribers.append( (service_name,cb,args,kwargs) )
         self._subscribed_service_names.add(service_name)
         self._maybe_subscribe()
@@ -349,5 +370,7 @@ class IntroducerClient(service.Service, Referenceable):
             if service_name2 == service_name:
                 eventually(cb, key_s, ann, *args, **kwargs)
 
+        self._save_announcements()
+
     def connected_to_introducer(self):
         return bool(self._publisher)

src/allmydata/test/test_introducer.py

@@ -1,11 +1,12 @@
 
-import os, re, itertools
+import os, re, itertools, yaml
 from base64 import b32decode
 import simplejson
 
 from twisted.trial import unittest
 from twisted.internet import defer, address
 from twisted.python import log
+from twisted.python.filepath import FilePath
 
 from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue
 from twisted.application import service
@@ -91,7 +92,7 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
 
     def test_create(self):
         ic = IntroducerClient(None, "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq, FilePath("notexist.introduced.yaml"))
         self.failUnless(isinstance(ic, IntroducerClient))
 
     def test_listen(self):
@@ -123,7 +124,8 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
         i = IntroducerService()
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath("notexist.introduced.yaml"))
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
         keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
@@ -172,7 +174,8 @@ class Client(unittest.TestCase):
     def test_duplicate_receive_v1(self):
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath("notexist.introduced.yaml"))
         announcements = []
         ic.subscribe_to("storage",
                         lambda key_s,ann: announcements.append(ann))
@@ -221,12 +224,14 @@ class Client(unittest.TestCase):
     def test_duplicate_receive_v2(self):
         ic1 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver23", "oldest_version", {}, fakeseq)
+                               "ver23", "oldest_version", {}, fakeseq,
+                               FilePath("notexist.introduced.yaml"))
         # we use a second client just to create a different-looking
         # announcement
         ic2 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver24","oldest_version",{}, fakeseq)
+                               "ver24","oldest_version",{}, fakeseq,
+                               FilePath("notexist.introduced.yaml"))
         announcements = []
         def _received(key_s, ann):
             announcements.append( (key_s, ann) )
@@ -329,7 +334,8 @@ class Client(unittest.TestCase):
         # not replace the other)
         ic = IntroducerClient(None,
                               "introducer.furl", u"my_nickname",
-                              "my_version", "oldest_version", {}, fakeseq)
+                              "my_version", "oldest_version", {}, fakeseq,
+                              FilePath("notexist.introduced.yaml"))
         announcements = []
         ic.subscribe_to("storage",
                         lambda key_s,ann: announcements.append(ann))
@@ -367,7 +373,8 @@ class Server(unittest.TestCase):
         i = IntroducerService()
         ic1 = IntroducerClient(None,
                                "introducer.furl", u"my_nickname",
-                               "ver23", "oldest_version", {}, realseq)
+                               "ver23", "oldest_version", {}, realseq,
+                               FilePath("notexist.introduced.yaml"))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
 
         privkey_s, _ = keyutil.make_keypair()
@@ -469,7 +476,8 @@ class Queue(SystemTestMixin, unittest.TestCase):
         tub2 = Tub()
         tub2.setServiceParent(self.parent)
         c = IntroducerClient(tub2, ifurl,
-                             u"nickname", "version", "oldest", {}, fakeseq)
+                             u"nickname", "version", "oldest", {}, fakeseq,
+                             FilePath("notexist.introduced.yaml"))
         furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
@@ -557,7 +565,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
                 c = IntroducerClient(tub, self.introducer_furl,
                                      NICKNAME % str(i),
                                      "version", "oldest",
-                                     {"component": "component-v1"}, fakeseq)
+                                     {"component": "component-v1"}, fakeseq,
+                                     FilePath("notexist.introduced.yaml"))
             received_announcements[c] = {}
             def got(key_s_or_tubid, ann, announcements, i):
                 if i == 0:
@@ -877,7 +886,7 @@ class ClientInfo(unittest.TestCase):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath("notexist.introduced.yaml"))
         #furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         #ann_s = make_ann_t(client_v2, furl1, None, 10)
         #introducer.remote_publish_v2(ann_s, Referenceable())
@@ -939,7 +948,7 @@ class Announcements(unittest.TestCase):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath("notexist.introduced.yaml"))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
         ann_s0 = make_ann_t(client_v2, furl1, None, 10)
@@ -961,7 +970,7 @@ class Announcements(unittest.TestCase):
         app_versions = {"whizzy": "fizzy"}
         client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
                                      "my_version", "oldest", app_versions,
-                                     fakeseq)
+                                     fakeseq, FilePath("notexist.introduced.yaml"))
         furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
         sk_s, vk_s = keyutil.make_keypair()
         sk, _ignored = keyutil.parse_privkey(sk_s)
@@ -998,6 +1007,78 @@ class Announcements(unittest.TestCase):
         self.failUnlessEqual(a[0].version, "my_version")
         self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
 
+    def test_client_cache_1(self):
+        basedir = "introducer/ClientSeqnums/test_client_cache_1"
+        fileutil.make_dirs(basedir)
+        cache_filepath = FilePath(os.path.join(basedir, "private", "introducer_cache.yaml"))
+
+        # if storage is enabled, the Client will publish its storage server
+        # during startup (although the announcement will wait in a queue
+        # until the introducer connection is established). To avoid getting
+        # confused by this, disable storage.
+        f = open(os.path.join(basedir, "tahoe.cfg"), "w")
+        f.write("[client]\n")
+        f.write("introducer.furl = nope\n")
+        f.write("[storage]\n")
+        f.write("enabled = false\n")
+        f.close()
+
+        c = TahoeClient(basedir)
+        ic = c.introducer_client
+        sk_s, vk_s = keyutil.make_keypair()
+        sk, _ignored = keyutil.parse_privkey(sk_s)
+        keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
+        furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
+        ann_t = make_ann_t(ic, furl1, sk, 1)
+
+        ic.got_announcements([ann_t])
+
+        # check the cache for the announcement
+        with cache_filepath.open() as f:
+            def constructor(loader, node):
+                return node.value
+            yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
+            announcements = yaml.safe_load(f)
+            f.close()
+
+        self.failUnlessEqual(len(announcements), 1)
+        self.failUnlessEqual("pub-" + announcements[0]['key_s'], vk_s)
+
+    def test_client_cache_2(self):
+        basedir = "introducer/ClientSeqnums/test_client_cache_2"
+        fileutil.make_dirs(basedir)
+        cache_filepath = FilePath(os.path.join(basedir, "private", "introducer_cache.yaml"))
+
+        # if storage is enabled, the Client will publish its storage server
+        # during startup (although the announcement will wait in a queue
+        # until the introducer connection is established). To avoid getting
+        # confused by this, disable storage.
+        f = open(os.path.join(basedir, "tahoe.cfg"), "w")
+        f.write("[client]\n")
+        f.write("introducer.furl = nope\n")
+        f.write("[storage]\n")
+        f.write("enabled = false\n")
+        f.close()
+
+        c = TahoeClient(basedir)
+        ic = c.introducer_client
+        furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@192.168.69.247:36106,127.0.0.1:36106/gydnpigj2ja2qr2srq4ikjwnl7xfgbra"
+        ann1 = (furl1, "storage", "RIStorage", "nick1", "ver23", "ver0")
+        ann = make_ann_t(ic, furl1, '', 2)
+        ic.got_announcements([ann])
+
+        # check the cache for the announcement
+        with cache_filepath.open() as f:
+            def constructor(loader, node):
+                return node.value
+            yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
+            announcements = yaml.safe_load(f)
+            f.close()
+
+        self.failUnlessEqual(len(announcements), 1)
+        self.failUnlessEqual(announcements[0]['key_s'], None)
+
+
 class ClientSeqnums(unittest.TestCase):
     def test_client(self):
         basedir = "introducer/ClientSeqnums/test_client"
@@ -1079,7 +1160,7 @@ class NonV1Server(SystemTestMixin, unittest.TestCase):
 
         c = IntroducerClient(tub, self.introducer_furl,
                              u"nickname-client", "version", "oldest", {},
-                             fakeseq)
+                             fakeseq, FilePath("notexist.introduced.yaml"))
         announcements = {}
         def got(key_s, ann):
             announcements[key_s] = ann