ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-21 02:01:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add IntroducerClient write-only yaml cache file

Browse Source 
this change also includes unit tests
This commit is contained in:
David Stainton 2016-05-10 20:19:35 +00:00
parent e3aef2b966
commit ae2b82a1f0
4 changed files with 125 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/allmydata/_auto_deps.py
View File

@ -87,6 +87,7 @@ install_requires = [
# * pyOpenSSL >= 0.14 is needed in order to avoid
# <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2474>.
"pyOpenSSL >= 0.14",
"PyYAML >= 3.11",
]
# Includes some indirect dependencies, but does not include allmydata.
@ -114,6 +115,7 @@ package_imports = [
('six', 'six'),
('enum34', 'enum'),
('pycparser', 'pycparser'),
('PyYAML', 'yaml'),
]
# Dependencies for which we don't know how to get a version number at run-time.

4
src/allmydata/client.py
View File

@ -6,6 +6,7 @@ from zope.interface import implements
from twisted.internet import reactor, defer
from twisted.application import service
from twisted.application.internet import TimerService
from twisted.python.filepath import FilePath
from pycryptopp.publickey import rsa
import allmydata
@ -171,12 +172,13 @@ class Client(node.Node, pollmixin.PollMixin):
def init_introducer_client(self):
self.introducer_furl = self.get_config("client", "introducer.furl")
introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml"))
ic = IntroducerClient(self.tub, self.introducer_furl,
self.nickname,
str(allmydata.__full_version__),
str(self.OLDEST_SUPPORTED_VERSION),
self.get_app_versions(),
self._sequencer)
self._sequencer, introducer_cache_filepath)
self.introducer_client = ic
ic.setServiceParent(self)

27
src/allmydata/introducer/client.py
View File

@ -1,5 +1,5 @@
import time
import time, yaml
from zope.interface import implements
from twisted.application import service
from foolscap.api import Referenceable, eventually, RemoteInterface
@ -47,7 +47,7 @@ class IntroducerClient(service.Service, Referenceable):
def __init__(self, tub, introducer_furl,
nickname, my_version, oldest_supported,
app_versions, sequencer):
app_versions, sequencer, cache_filepath):
self._tub = tub
self.introducer_furl = introducer_furl
@ -57,6 +57,7 @@ class IntroducerClient(service.Service, Referenceable):
self._oldest_supported = oldest_supported
self._app_versions = app_versions
self._sequencer = sequencer
self._cache_filepath = cache_filepath
self._my_subscriber_info = { "version": 0,
"nickname": self._nickname,
@ -112,6 +113,25 @@ class IntroducerClient(service.Service, Referenceable):
level=log.WEIRD, failure=failure, umid="c5MqUQ")
d = self._tub.getReference(self.introducer_furl)
d.addErrback(connect_failed)
def remove_cache(result):
try:
self._cache_filepath.remove()
except OSError, e:
pass
return result
d.addCallback(remove_cache)
def _save_announcements(self):
announcements = []
for _, value in self._inbound_announcements.items():
ann, key_s, time_stamp = value
server_params = {
"ann" : ann,
"key_s" : key_s,
}
announcements.append(server_params)
announcement_cache_yaml = yaml.dump(announcements)
self._cache_filepath.setContent(announcement_cache_yaml)
def _got_introducer(self, publisher):
self.log("connected to introducer, getting versions")
@ -148,6 +168,7 @@ class IntroducerClient(service.Service, Referenceable):
return log.msg(*args, **kwargs)
def subscribe_to(self, service_name, cb, *args, **kwargs):
self._got_announcement_cb = cb
self._local_subscribers.append( (service_name,cb,args,kwargs) )
self._subscribed_service_names.add(service_name)
self._maybe_subscribe()
@ -349,5 +370,7 @@ class IntroducerClient(service.Service, Referenceable):
if service_name2 == service_name:
eventually(cb, key_s, ann, *args, **kwargs)
self._save_announcements()
def connected_to_introducer(self):
return bool(self._publisher)

109
src/allmydata/test/test_introducer.py
View File

@ -1,11 +1,12 @@
import os, re, itertools
import os, re, itertools, yaml
from base64 import b32decode
import simplejson
from twisted.trial import unittest
from twisted.internet import defer, address
from twisted.python import log
from twisted.python.filepath import FilePath
from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue
from twisted.application import service
@ -91,7 +92,7 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
def test_create(self):
ic = IntroducerClient(None, "introducer.furl", u"my_nickname",
"my_version", "oldest_version", {}, fakeseq)
"my_version", "oldest_version", {}, fakeseq, FilePath("notexist.introduced.yaml"))
self.failUnless(isinstance(ic, IntroducerClient))
def test_listen(self):
@ -123,7 +124,8 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
i = IntroducerService()
ic = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"my_version", "oldest_version", {}, fakeseq)
"my_version", "oldest_version", {}, fakeseq,
FilePath("notexist.introduced.yaml"))
sk_s, vk_s = keyutil.make_keypair()
sk, _ignored = keyutil.parse_privkey(sk_s)
keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
@ -172,7 +174,8 @@ class Client(unittest.TestCase):
def test_duplicate_receive_v1(self):
ic = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"my_version", "oldest_version", {}, fakeseq)
"my_version", "oldest_version", {}, fakeseq,
FilePath("notexist.introduced.yaml"))
announcements = []
ic.subscribe_to("storage",
lambda key_s,ann: announcements.append(ann))
@ -221,12 +224,14 @@ class Client(unittest.TestCase):
def test_duplicate_receive_v2(self):
ic1 = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"ver23", "oldest_version", {}, fakeseq)
"ver23", "oldest_version", {}, fakeseq,
FilePath("notexist.introduced.yaml"))
# we use a second client just to create a different-looking
# announcement
ic2 = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"ver24","oldest_version",{}, fakeseq)
"ver24","oldest_version",{}, fakeseq,
FilePath("notexist.introduced.yaml"))
announcements = []
def _received(key_s, ann):
announcements.append( (key_s, ann) )
@ -329,7 +334,8 @@ class Client(unittest.TestCase):
# not replace the other)
ic = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"my_version", "oldest_version", {}, fakeseq)
"my_version", "oldest_version", {}, fakeseq,
FilePath("notexist.introduced.yaml"))
announcements = []
ic.subscribe_to("storage",
lambda key_s,ann: announcements.append(ann))
@ -367,7 +373,8 @@ class Server(unittest.TestCase):
i = IntroducerService()
ic1 = IntroducerClient(None,
"introducer.furl", u"my_nickname",
"ver23", "oldest_version", {}, realseq)
"ver23", "oldest_version", {}, realseq,
FilePath("notexist.introduced.yaml"))
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
privkey_s, _ = keyutil.make_keypair()
@ -469,7 +476,8 @@ class Queue(SystemTestMixin, unittest.TestCase):
tub2 = Tub()
tub2.setServiceParent(self.parent)
c = IntroducerClient(tub2, ifurl,
u"nickname", "version", "oldest", {}, fakeseq)
u"nickname", "version", "oldest", {}, fakeseq,
FilePath("notexist.introduced.yaml"))
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
sk_s, vk_s = keyutil.make_keypair()
sk, _ignored = keyutil.parse_privkey(sk_s)
@ -557,7 +565,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
c = IntroducerClient(tub, self.introducer_furl,
NICKNAME % str(i),
"version", "oldest",
{"component": "component-v1"}, fakeseq)
{"component": "component-v1"}, fakeseq,
FilePath("notexist.introduced.yaml"))
received_announcements[c] = {}
def got(key_s_or_tubid, ann, announcements, i):
if i == 0:
@ -877,7 +886,7 @@ class ClientInfo(unittest.TestCase):
app_versions = {"whizzy": "fizzy"}
client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
"my_version", "oldest", app_versions,
fakeseq)
fakeseq, FilePath("notexist.introduced.yaml"))
#furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
#ann_s = make_ann_t(client_v2, furl1, None, 10)
#introducer.remote_publish_v2(ann_s, Referenceable())
@ -939,7 +948,7 @@ class Announcements(unittest.TestCase):
app_versions = {"whizzy": "fizzy"}
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
"my_version", "oldest", app_versions,
fakeseq)
fakeseq, FilePath("notexist.introduced.yaml"))
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
ann_s0 = make_ann_t(client_v2, furl1, None, 10)
@ -961,7 +970,7 @@ class Announcements(unittest.TestCase):
app_versions = {"whizzy": "fizzy"}
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
"my_version", "oldest", app_versions,
fakeseq)
fakeseq, FilePath("notexist.introduced.yaml"))
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
sk_s, vk_s = keyutil.make_keypair()
sk, _ignored = keyutil.parse_privkey(sk_s)
@ -998,6 +1007,78 @@ class Announcements(unittest.TestCase):
self.failUnlessEqual(a[0].version, "my_version")
self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def test_client_cache_1(self):
basedir = "introducer/ClientSeqnums/test_client_cache_1"
fileutil.make_dirs(basedir)
cache_filepath = FilePath(os.path.join(basedir, "private", "introducer_cache.yaml"))
# if storage is enabled, the Client will publish its storage server
# during startup (although the announcement will wait in a queue
# until the introducer connection is established). To avoid getting
# confused by this, disable storage.
f = open(os.path.join(basedir, "tahoe.cfg"), "w")
f.write("[client]\n")
f.write("introducer.furl = nope\n")
f.write("[storage]\n")
f.write("enabled = false\n")
f.close()
c = TahoeClient(basedir)
ic = c.introducer_client
sk_s, vk_s = keyutil.make_keypair()
sk, _ignored = keyutil.parse_privkey(sk_s)
keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
ann_t = make_ann_t(ic, furl1, sk, 1)
ic.got_announcements([ann_t])
# check the cache for the announcement
with cache_filepath.open() as f:
def constructor(loader, node):
return node.value
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
announcements = yaml.safe_load(f)
f.close()
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual("pub-" + announcements[0]['key_s'], vk_s)
def test_client_cache_2(self):
basedir = "introducer/ClientSeqnums/test_client_cache_2"
fileutil.make_dirs(basedir)
cache_filepath = FilePath(os.path.join(basedir, "private", "introducer_cache.yaml"))
# if storage is enabled, the Client will publish its storage server
# during startup (although the announcement will wait in a queue
# until the introducer connection is established). To avoid getting
# confused by this, disable storage.
f = open(os.path.join(basedir, "tahoe.cfg"), "w")
f.write("[client]\n")
f.write("introducer.furl = nope\n")
f.write("[storage]\n")
f.write("enabled = false\n")
f.close()
c = TahoeClient(basedir)
ic = c.introducer_client
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@192.168.69.247:36106,127.0.0.1:36106/gydnpigj2ja2qr2srq4ikjwnl7xfgbra"
ann1 = (furl1, "storage", "RIStorage", "nick1", "ver23", "ver0")
ann = make_ann_t(ic, furl1, '', 2)
ic.got_announcements([ann])
# check the cache for the announcement
with cache_filepath.open() as f:
def constructor(loader, node):
return node.value
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
announcements = yaml.safe_load(f)
f.close()
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual(announcements[0]['key_s'], None)
class ClientSeqnums(unittest.TestCase):
def test_client(self):
basedir = "introducer/ClientSeqnums/test_client"
@ -1079,7 +1160,7 @@ class NonV1Server(SystemTestMixin, unittest.TestCase):
c = IntroducerClient(tub, self.introducer_furl,
u"nickname-client", "version", "oldest", {},
fakeseq)
fakeseq, FilePath("notexist.introduced.yaml"))
announcements = {}
def got(key_s, ann):
announcements[key_s] = ann