ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-18 17:00:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Teach tub to start synchronously even with ip autodetect

Browse Source 
* remove when_tub_ready() from all code
* synchronous-ify all node/client/introducer startup code

refs ticket:2491
This commit is contained in:
Brian Warner 2016-04-26 21:54:45 -07:00
parent ffc2f48cfe
commit 96c625920c
9 changed files with 84 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/allmydata/client.py
View File

@ -131,6 +131,8 @@ class Client(node.Node, pollmixin.PollMixin):
def __init__(self, basedir="."):
node.Node.__init__(self, basedir)
# All tub.registerReference must happen *after* we upcall, since
# that's what does tub.setLocation()
self.started_timestamp = time.time()
self.logSource="Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
@ -188,15 +190,7 @@ class Client(node.Node, pollmixin.PollMixin):
self.get_app_versions(),
self._sequencer)
self.introducer_client = ic
# hold off on starting the IntroducerClient until our tub has been
# started, so we'll have a useful address on our RemoteReference, so
# that the introducer's status page will show us.
d = self.when_tub_ready()
def _start_introducer_client(res):
ic.setServiceParent(self)
d.addCallback(_start_introducer_client)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="URyI5w")
ic.setServiceParent(self)
def init_stats_provider(self):
gatherer_furl = self.get_config("client", "stats_gatherer.furl", None)
@ -309,18 +303,12 @@ class Client(node.Node, pollmixin.PollMixin):
expiration_sharetypes=expiration_sharetypes)
self.add_service(ss)
d = self.when_tub_ready()
# we can't do registerReference until the Tub is ready
def _publish(res):
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._node_key)
d.addCallback(_publish)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="aLGBKw")
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._node_key)
def init_client(self):
helper_furl = self.get_config("client", "helper.furl", None)
@ -433,41 +421,26 @@ class Client(node.Node, pollmixin.PollMixin):
return self.history
def init_control(self):
d = self.when_tub_ready()
def _publish(res):
c = ControlServer()
c.setServiceParent(self)
control_url = self.tub.registerReference(c)
self.write_private_config("control.furl", control_url + "\n")
d.addCallback(_publish)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="d3tNXA")
c = ControlServer()
c.setServiceParent(self)
control_url = self.tub.registerReference(c)
self.write_private_config("control.furl", control_url + "\n")
def init_helper(self):
d = self.when_tub_ready()
def _publish(self):
self.helper = Helper(os.path.join(self.basedir, "helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created
# by the helper. BASEDIR/helper.furl is consumed by the client
# who wants to use the helper. I like having the filename be the
# same, since that makes 'cp' work smoothly, but the difference
# between config inputs and generated outputs is hard to see.
helper_furlfile = os.path.join(self.basedir,
"private", "helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
d.addCallback(_publish)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="K0mW5w")
self.helper = Helper(os.path.join(self.basedir, "helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created by
# the helper. BASEDIR/helper.furl is consumed by the client who wants
# to use the helper. I like having the filename be the same, since
# that makes 'cp' work smoothly, but the difference between config
# inputs and generated outputs is hard to see.
helper_furlfile = os.path.join(self.basedir,
"private", "helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
def init_key_gen(self, key_gen_furl):
d = self.when_tub_ready()
def _subscribe(self):
self.tub.connectTo(key_gen_furl, self._got_key_generator)
d.addCallback(_subscribe)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="z9DMzw")
self.tub.connectTo(key_gen_furl, self._got_key_generator)
def _got_key_generator(self, key_generator):
self._key_generator.set_remote_generator(key_generator)

13
src/allmydata/introducer/server.py
View File

@ -48,15 +48,10 @@ class IntroducerNode(node.Node):
"""
raise FurlFileConflictError(textwrap.dedent(msg))
os.rename(old_public_fn, private_fn)
d = self.when_tub_ready()
def _publish(res):
furl = self.tub.registerReference(introducerservice,
furlFile=private_fn)
self.log(" introducer is at %s" % furl, umid="qF2L9A")
self.introducer_url = furl # for tests
d.addCallback(_publish)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="UaNs9A")
furl = self.tub.registerReference(introducerservice,
furlFile=private_fn)
self.log(" introducer is at %s" % furl, umid="qF2L9A")
self.introducer_url = furl # for tests
def init_web(self, webport):
self.log("init_web(webport=%s)", args=(webport,), umid="2bUygA")

26
src/allmydata/node.py
View File

@ -3,13 +3,12 @@ from base64 import b32decode, b32encode
from twisted.python import log as twlog
from twisted.application import service
from twisted.internet import reactor
from foolscap.api import Tub, eventually, app_versions
from foolscap.api import Tub, app_versions
import foolscap.logging.log
from allmydata import get_package_versions, get_package_versions_string
from allmydata.util import log
from allmydata.util import fileutil, iputil, observer
from allmydata.util.assertutil import precondition, _assert
from allmydata.util import fileutil, iputil
from allmydata.util.assertutil import _assert
from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.util.encodingutil import get_filesystem_encoding, quote_output
from allmydata.util import configutil
@ -74,7 +73,6 @@ class Node(service.MultiService):
service.MultiService.__init__(self)
self.basedir = abspath_expanduser_unicode(unicode(basedir))
self._portnumfile = os.path.join(self.basedir, self.PORTNUMFILE)
self._tub_ready_observerlist = observer.OneShotObserverList()
fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700)
open(os.path.join(self.basedir, "private", "README"), "w").write(PRIV_README)
@ -228,6 +226,7 @@ class Node(service.MultiService):
self.tub.setLocation(location)
self.log("Tub location set to %s" % (location,))
# the Tub is now ready for tub.registerReference()
self.tub.setServiceParent(self)
def get_app_versions(self):
@ -321,25 +320,13 @@ class Node(service.MultiService):
os.chmod("twistd.pid", 0644)
except EnvironmentError:
pass
# Delay until the reactor is running.
eventually(self._startService)
def _startService(self):
precondition(reactor.running)
self.log("Node._startService")
service.MultiService.startService(self)
self.log("%s running" % self.NODETYPE)
self._tub_ready_observerlist.fire(self)
def stopService(self):
self.log("Node.stopService")
d = self._tub_ready_observerlist.when_fired()
def _really_stopService(ignored):
self.log("Node._really_stopService")
return service.MultiService.stopService(self)
d.addCallback(_really_stopService)
return d
return service.MultiService.stopService(self)
def shutdown(self):
"""Shut down the node. Returns a Deferred that fires (with None) when
@ -375,9 +362,6 @@ class Node(service.MultiService):
def log(self, *args, **kwargs):
return log.msg(*args, **kwargs)
def when_tub_ready(self):
return self._tub_ready_observerlist.when_fired()
def add_service(self, s):
s.setServiceParent(self)
return s

9
src/allmydata/stats.py
View File

@ -148,12 +148,9 @@ class StatsProvider(Referenceable, service.MultiService):
def startService(self):
if self.node and self.gatherer_furl:
d = self.node.when_tub_ready()
def connect(junk):
nickname_utf8 = self.node.nickname.encode("utf-8")
self.node.tub.connectTo(self.gatherer_furl,
self._connected, nickname_utf8)
d.addCallback(connect)
nickname_utf8 = self.node.nickname.encode("utf-8")
self.node.tub.connectTo(self.gatherer_furl,
self._connected, nickname_utf8)
service.MultiService.startService(self)
def count(self, name, delta=1):

13
src/allmydata/test/check_memory.py
View File

@ -116,10 +116,8 @@ class SystemFramework(pollmixin.PollMixin):
#print "STARTING"
self.stats = {}
self.statsfile = open(os.path.join(self.basedir, "stats.out"), "a")
d = self.make_introducer()
def _more(res):
return self.start_client()
d.addCallback(_more)
self.make_introducer()
d = self.start_client()
def _record_control_furl(control_furl):
self.control_furl = control_furl
#print "OBTAINING '%s'" % (control_furl,)
@ -174,12 +172,7 @@ class SystemFramework(pollmixin.PollMixin):
os.mkdir(iv_basedir)
iv = introducer.IntroducerNode(basedir=iv_basedir)
self.introducer = self.add_service(iv)
d = self.introducer.when_tub_ready()
def _introducer_ready(res):
q = self.introducer
self.introducer_furl = q.introducer_url
d.addCallback(_introducer_ready)
return d
self.introducer_furl = self.introducer.introducer_url
def make_nodes(self):
self.nodes = []

45
src/allmydata/test/common.py
View File

@ -481,8 +481,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
f.close()
iv = IntroducerNode(basedir=iv_dir)
self.introducer = self.add_service(iv)
d = self.introducer.when_tub_ready()
d.addCallback(self._get_introducer_web)
self._get_introducer_web()
d = defer.succeed(None)
if use_stats_gatherer:
d.addCallback(self._set_up_stats_gatherer)
if use_key_generator:
@ -492,7 +492,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
d.addCallback(self._grab_stats)
return d
def _get_introducer_web(self, res):
def _get_introducer_web(self):
f = open(os.path.join(self.getdir("introducer"), "node.url"), "r")
self.introweb_url = f.read().strip()
f.close()
@ -591,27 +591,25 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
c = self.add_service(client.Client(basedir=basedirs[0]))
self.clients.append(c)
c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
d = c.when_tub_ready()
def _ready(res):
f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
helper_furl = f.read()
f.close()
self.helper_furl = helper_furl
if self.numclients >= 4:
f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+')
f.write(
"[client]\n"
"helper.furl = %s\n" % helper_furl)
f.close()
# this starts the rest of the clients
for i in range(1, self.numclients):
c = self.add_service(client.Client(basedir=basedirs[i]))
self.clients.append(c)
c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
log.msg("STARTING")
return self.wait_for_connections()
d.addCallback(_ready)
f = open(os.path.join(basedirs[0],"private","helper.furl"), "r")
helper_furl = f.read()
f.close()
self.helper_furl = helper_furl
if self.numclients >= 4:
f = open(os.path.join(basedirs[3], 'tahoe.cfg'), 'ab+')
f.write(
"[client]\n"
"helper.furl = %s\n" % helper_furl)
f.close()
# this starts the rest of the clients
for i in range(1, self.numclients):
c = self.add_service(client.Client(basedir=basedirs[i]))
self.clients.append(c)
c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
log.msg("STARTING")
d = self.wait_for_connections()
def _connected(res):
log.msg("CONNECTED")
# now find out where the web port was
@ -643,7 +641,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
self.clients[num] = new_c
new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
self.add_service(new_c)
return new_c.when_tub_ready()
d.addCallback(_stopped)
d.addCallback(lambda res: self.wait_for_connections())
def _maybe_get_webport(res):

2
src/allmydata/test/no_network.py
View File

@ -184,8 +184,6 @@ class NoNetworkClient(Client):
service.MultiService.startService(self)
def stopService(self):
service.MultiService.stopService(self)
def when_tub_ready(self):
raise NotImplementedError("NoNetworkClient has no Tub")
def init_control(self):
pass
def init_helper(self):

58
src/allmydata/test/test_introducer.py
View File

@ -34,45 +34,33 @@ class Node(testutil.SignalMixin, testutil.ReallyEqualMixin, unittest.TestCase):
os.mkdir(basedir)
public_fn = os.path.join(basedir, "introducer.furl")
private_fn = os.path.join(basedir, "private", "introducer.furl")
q1 = IntroducerNode(basedir)
d = fireEventually(None)
d.addCallback(lambda res: q1.startService())
d.addCallback(lambda res: q1.when_tub_ready())
d.addCallback(lambda res: q1.stopService())
d.addCallback(flushEventualQueue)
def _check_furl(res):
# new nodes create unguessable furls in private/introducer.furl
ifurl = fileutil.read(private_fn)
self.failUnless(ifurl)
ifurl = ifurl.strip()
self.failIf(ifurl.endswith("/introducer"), ifurl)
del q1
# new nodes create unguessable furls in private/introducer.furl
ifurl = fileutil.read(private_fn)
self.failUnless(ifurl)
ifurl = ifurl.strip()
self.failIf(ifurl.endswith("/introducer"), ifurl)
# old nodes created guessable furls in BASEDIR/introducer.furl
guessable = ifurl[:ifurl.rfind("/")] + "/introducer"
fileutil.write(public_fn, guessable+"\n", mode="w") # text
# old nodes created guessable furls in BASEDIR/introducer.furl
guessable = ifurl[:ifurl.rfind("/")] + "/introducer"
fileutil.write(public_fn, guessable+"\n", mode="w") # text
# if we see both files, throw an error
self.failUnlessRaises(FurlFileConflictError,
IntroducerNode, basedir)
# if we see both files, throw an error
self.failUnlessRaises(FurlFileConflictError,
IntroducerNode, basedir)
# when we see only the public one, move it to private/ and use
# the existing furl instead of creating a new one
os.unlink(private_fn)
q2 = IntroducerNode(basedir)
d2 = fireEventually(None)
d2.addCallback(lambda res: q2.startService())
d2.addCallback(lambda res: q2.when_tub_ready())
d2.addCallback(lambda res: q2.stopService())
d2.addCallback(flushEventualQueue)
def _check_furl2(res):
self.failIf(os.path.exists(public_fn))
ifurl2 = fileutil.read(private_fn)
self.failUnless(ifurl2)
self.failUnlessEqual(ifurl2.strip(), guessable)
d2.addCallback(_check_furl2)
return d2
d.addCallback(_check_furl)
return d
# when we see only the public one, move it to private/ and use
# the existing furl instead of creating a new one
os.unlink(private_fn)
q2 = IntroducerNode(basedir)
del q2
self.failIf(os.path.exists(public_fn))
ifurl2 = fileutil.read(private_fn)
self.failUnless(ifurl2)
self.failUnlessEqual(ifurl2.strip(), guessable)
def test_web_static(self):
basedir = u"introducer.Node.test_web_static"

1
src/allmydata/test/test_web.py
View File

@ -4465,7 +4465,6 @@ class IntroducerWeb(unittest.TestCase):
d = fireEventually(None)
d.addCallback(lambda ign: self.node.startService())
d.addCallback(lambda ign: self.node.when_tub_ready())
d.addCallback(lambda ign: self.GET("/"))
def _check(res):