mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-02-21 02:01:31 +00:00
Merge branch 'pr278': write (but don't read) YAML cache
refs ticket:2759
This commit is contained in:
Brian Warner
commit
b49b409c32
@ -87,6 +87,7 @@ install_requires = [
|
||||
# * pyOpenSSL >= 0.14 is needed in order to avoid
|
||||
# <https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2474>.
|
||||
"pyOpenSSL >= 0.14",
|
||||
"PyYAML >= 3.11",
|
||||
]
|
||||
|
||||
# Includes some indirect dependencies, but does not include allmydata.
|
||||
@ -114,6 +115,7 @@ package_imports = [
|
||||
('six', 'six'),
|
||||
('enum34', 'enum'),
|
||||
('pycparser', 'pycparser'),
|
||||
('PyYAML', 'yaml'),
|
||||
]
|
||||
|
||||
# Dependencies for which we don't know how to get a version number at run-time.
|
||||
|
||||
@ -6,6 +6,7 @@ from zope.interface import implements
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.application import service
|
||||
from twisted.application.internet import TimerService
|
||||
from twisted.python.filepath import FilePath
|
||||
from pycryptopp.publickey import rsa
|
||||
|
||||
import allmydata
|
||||
@ -171,12 +172,13 @@ class Client(node.Node, pollmixin.PollMixin):
|
||||
|
||||
def init_introducer_client(self):
|
||||
self.introducer_furl = self.get_config("client", "introducer.furl")
|
||||
introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_cache.yaml"))
|
||||
ic = IntroducerClient(self.tub, self.introducer_furl,
|
||||
self.nickname,
|
||||
str(allmydata.__full_version__),
|
||||
str(self.OLDEST_SUPPORTED_VERSION),
|
||||
self.get_app_versions(),
|
||||
self._sequencer)
|
||||
self._sequencer, introducer_cache_filepath)
|
||||
self.introducer_client = ic
|
||||
ic.setServiceParent(self)
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
|
||||
import time
|
||||
import time, yaml
|
||||
from zope.interface import implements
|
||||
from twisted.application import service
|
||||
from foolscap.api import Referenceable, eventually, RemoteInterface
|
||||
@ -47,7 +47,7 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
|
||||
def __init__(self, tub, introducer_furl,
|
||||
nickname, my_version, oldest_supported,
|
||||
app_versions, sequencer):
|
||||
app_versions, sequencer, cache_filepath):
|
||||
self._tub = tub
|
||||
self.introducer_furl = introducer_furl
|
||||
|
||||
@ -57,6 +57,7 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
self._oldest_supported = oldest_supported
|
||||
self._app_versions = app_versions
|
||||
self._sequencer = sequencer
|
||||
self._cache_filepath = cache_filepath
|
||||
|
||||
self._my_subscriber_info = { "version": 0,
|
||||
"nickname": self._nickname,
|
||||
@ -113,6 +114,18 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
d = self._tub.getReference(self.introducer_furl)
|
||||
d.addErrback(connect_failed)
|
||||
|
||||
def _save_announcements(self):
|
||||
announcements = []
|
||||
for _, value in self._inbound_announcements.items():
|
||||
ann, key_s, time_stamp = value
|
||||
server_params = {
|
||||
"ann" : ann,
|
||||
"key_s" : key_s,
|
||||
}
|
||||
announcements.append(server_params)
|
||||
announcement_cache_yaml = yaml.dump(announcements)
|
||||
self._cache_filepath.setContent(announcement_cache_yaml)
|
||||
|
||||
def _got_introducer(self, publisher):
|
||||
self.log("connected to introducer, getting versions")
|
||||
default = { "http://allmydata.org/tahoe/protocols/introducer/v1":
|
||||
@ -343,6 +356,7 @@ class IntroducerClient(service.Service, Referenceable):
|
||||
parent=lp2, level=log.NOISY)
|
||||
|
||||
self._inbound_announcements[index] = (ann, key_s, time.time())
|
||||
self._save_announcements()
|
||||
# note: we never forget an index, but we might update its value
|
||||
|
||||
for (service_name2,cb,args,kwargs) in self._local_subscribers:
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
|
||||
import os, re, itertools
|
||||
import os, re, itertools, yaml
|
||||
from base64 import b32decode
|
||||
import simplejson
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.internet import defer, address
|
||||
from twisted.python import log
|
||||
from twisted.python.filepath import FilePath
|
||||
|
||||
from foolscap.api import Tub, Referenceable, fireEventually, flushEventualQueue
|
||||
from twisted.application import service
|
||||
@ -91,7 +92,8 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
|
||||
|
||||
def test_create(self):
|
||||
ic = IntroducerClient(None, "introducer.furl", u"my_nickname",
|
||||
"my_version", "oldest_version", {}, fakeseq)
|
||||
"my_version", "oldest_version", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
self.failUnless(isinstance(ic, IntroducerClient))
|
||||
|
||||
def test_listen(self):
|
||||
@ -123,7 +125,8 @@ class Introducer(ServiceMixin, unittest.TestCase, pollmixin.PollMixin):
|
||||
i = IntroducerService()
|
||||
ic = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"my_version", "oldest_version", {}, fakeseq)
|
||||
"my_version", "oldest_version", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
sk_s, vk_s = keyutil.make_keypair()
|
||||
sk, _ignored = keyutil.parse_privkey(sk_s)
|
||||
keyid = keyutil.remove_prefix(vk_s, "pub-v0-")
|
||||
@ -172,7 +175,8 @@ class Client(unittest.TestCase):
|
||||
def test_duplicate_receive_v1(self):
|
||||
ic = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"my_version", "oldest_version", {}, fakeseq)
|
||||
"my_version", "oldest_version", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
announcements = []
|
||||
ic.subscribe_to("storage",
|
||||
lambda key_s,ann: announcements.append(ann))
|
||||
@ -221,12 +225,14 @@ class Client(unittest.TestCase):
|
||||
def test_duplicate_receive_v2(self):
|
||||
ic1 = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"ver23", "oldest_version", {}, fakeseq)
|
||||
"ver23", "oldest_version", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
# we use a second client just to create a different-looking
|
||||
# announcement
|
||||
ic2 = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"ver24","oldest_version",{}, fakeseq)
|
||||
"ver24","oldest_version",{}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
announcements = []
|
||||
def _received(key_s, ann):
|
||||
announcements.append( (key_s, ann) )
|
||||
@ -329,7 +335,8 @@ class Client(unittest.TestCase):
|
||||
# not replace the other)
|
||||
ic = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"my_version", "oldest_version", {}, fakeseq)
|
||||
"my_version", "oldest_version", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
announcements = []
|
||||
ic.subscribe_to("storage",
|
||||
lambda key_s,ann: announcements.append(ann))
|
||||
@ -367,7 +374,8 @@ class Server(unittest.TestCase):
|
||||
i = IntroducerService()
|
||||
ic1 = IntroducerClient(None,
|
||||
"introducer.furl", u"my_nickname",
|
||||
"ver23", "oldest_version", {}, realseq)
|
||||
"ver23", "oldest_version", {}, realseq,
|
||||
FilePath(self.mktemp()))
|
||||
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
|
||||
|
||||
privkey_s, _ = keyutil.make_keypair()
|
||||
@ -469,7 +477,8 @@ class Queue(SystemTestMixin, unittest.TestCase):
|
||||
tub2 = Tub()
|
||||
tub2.setServiceParent(self.parent)
|
||||
c = IntroducerClient(tub2, ifurl,
|
||||
u"nickname", "version", "oldest", {}, fakeseq)
|
||||
u"nickname", "version", "oldest", {}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
|
||||
sk_s, vk_s = keyutil.make_keypair()
|
||||
sk, _ignored = keyutil.parse_privkey(sk_s)
|
||||
@ -557,7 +566,8 @@ class SystemTest(SystemTestMixin, unittest.TestCase):
|
||||
c = IntroducerClient(tub, self.introducer_furl,
|
||||
NICKNAME % str(i),
|
||||
"version", "oldest",
|
||||
{"component": "component-v1"}, fakeseq)
|
||||
{"component": "component-v1"}, fakeseq,
|
||||
FilePath(self.mktemp()))
|
||||
received_announcements[c] = {}
|
||||
def got(key_s_or_tubid, ann, announcements, i):
|
||||
if i == 0:
|
||||
@ -877,7 +887,7 @@ class ClientInfo(unittest.TestCase):
|
||||
app_versions = {"whizzy": "fizzy"}
|
||||
client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
|
||||
"my_version", "oldest", app_versions,
|
||||
fakeseq)
|
||||
fakeseq, FilePath(self.mktemp()))
|
||||
#furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
|
||||
#ann_s = make_ann_t(client_v2, furl1, None, 10)
|
||||
#introducer.remote_publish_v2(ann_s, Referenceable())
|
||||
@ -939,7 +949,7 @@ class Announcements(unittest.TestCase):
|
||||
app_versions = {"whizzy": "fizzy"}
|
||||
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
|
||||
"my_version", "oldest", app_versions,
|
||||
fakeseq)
|
||||
fakeseq, FilePath(self.mktemp()))
|
||||
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
|
||||
tubid = "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
|
||||
ann_s0 = make_ann_t(client_v2, furl1, None, 10)
|
||||
@ -961,7 +971,7 @@ class Announcements(unittest.TestCase):
|
||||
app_versions = {"whizzy": "fizzy"}
|
||||
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
|
||||
"my_version", "oldest", app_versions,
|
||||
fakeseq)
|
||||
fakeseq, FilePath(self.mktemp()))
|
||||
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
|
||||
sk_s, vk_s = keyutil.make_keypair()
|
||||
sk, _ignored = keyutil.parse_privkey(sk_s)
|
||||
@ -998,6 +1008,45 @@ class Announcements(unittest.TestCase):
|
||||
self.failUnlessEqual(a[0].version, "my_version")
|
||||
self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
|
||||
|
||||
def test_client_cache_1(self):
|
||||
basedir = "introducer/ClientSeqnums/test_client_cache_1"
|
||||
fileutil.make_dirs(basedir)
|
||||
cache_filepath = FilePath(os.path.join(basedir, "private",
|
||||
"introducer_cache.yaml"))
|
||||
|
||||
# if storage is enabled, the Client will publish its storage server
|
||||
# during startup (although the announcement will wait in a queue
|
||||
# until the introducer connection is established). To avoid getting
|
||||
# confused by this, disable storage.
|
||||
f = open(os.path.join(basedir, "tahoe.cfg"), "w")
|
||||
f.write("[client]\n")
|
||||
f.write("introducer.furl = nope\n")
|
||||
f.write("[storage]\n")
|
||||
f.write("enabled = false\n")
|
||||
f.close()
|
||||
|
||||
c = TahoeClient(basedir)
|
||||
ic = c.introducer_client
|
||||
sk_s, vk_s = keyutil.make_keypair()
|
||||
sk, _ignored = keyutil.parse_privkey(sk_s)
|
||||
keyutil.remove_prefix(vk_s, "pub-v0-")
|
||||
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
|
||||
ann_t = make_ann_t(ic, furl1, sk, 1)
|
||||
|
||||
ic.got_announcements([ann_t])
|
||||
|
||||
# check the cache for the announcement
|
||||
with cache_filepath.open() as f:
|
||||
def constructor(loader, node):
|
||||
return node.value
|
||||
yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", constructor)
|
||||
announcements = yaml.safe_load(f)
|
||||
f.close()
|
||||
|
||||
self.failUnlessEqual(len(announcements), 1)
|
||||
self.failUnlessEqual("pub-" + announcements[0]['key_s'], vk_s)
|
||||
|
||||
|
||||
class ClientSeqnums(unittest.TestCase):
|
||||
def test_client(self):
|
||||
basedir = "introducer/ClientSeqnums/test_client"
|
||||
@ -1079,7 +1128,7 @@ class NonV1Server(SystemTestMixin, unittest.TestCase):
|
||||
|
||||
c = IntroducerClient(tub, self.introducer_furl,
|
||||
u"nickname-client", "version", "oldest", {},
|
||||
fakeseq)
|
||||
fakeseq, FilePath(self.mktemp()))
|
||||
announcements = {}
|
||||
def got(key_s, ann):
|
||||
announcements[key_s] = ann
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user