ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-18 17:00:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge PR417: rewrite tahoe start/stop/daemonize

Browse Source 
* refs ticket:1148 (splits up startstop_node, improves coverage)
* refs ticket:275 ('start' probably doesn't exit until furl is written)
* refs ticket:1121 (probably improves coverage)
* refs ticket:1377 (probably fixed)
* refs ticket:2149 (might influence, probably won't fix)
* refs ticket:719 (probably improved)
This commit is contained in:
Brian Warner 2017-09-19 09:57:59 -07:00
commit 04b34b6fd2
12 changed files with 915 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
docs/frontends/CLI.rst
View File

@ -83,12 +83,13 @@ the command line.
Node Management
===============
"``tahoe create-node [NODEDIR]``" is the basic make-a-new-node command. It
creates a new directory and populates it with files that will allow the
"``tahoe start``" command to use it later on. This command creates nodes that
have client functionality (upload/download files), web API services
(controlled by the '[node]web.port' configuration), and storage services
(unless ``--no-storage`` is specified).
"``tahoe create-node [NODEDIR]``" is the basic make-a-new-node
command. It creates a new directory and populates it with files that
will allow the "``tahoe start``" and related commands to use it later
on. ``tahoe create-node`` creates nodes that have client functionality
(upload/download files), web API services (controlled by the
'[node]web.port' configuration), and storage services (unless
``--no-storage`` is specified).
NODEDIR defaults to ``~/.tahoe/`` , and newly-created nodes default to
publishing a web server on port 3456 (limited to the loopback interface, at
@ -105,19 +106,34 @@ This node provides introduction services and nothing else. When started, this
node will produce a ``private/introducer.furl`` file, which should be
published to all clients.
"``tahoe run [NODEDIR]``" will start a previously-created node in the foreground.
"``tahoe start [NODEDIR]``" will launch a previously-created node. It will
launch the node into the background, using the standard Twisted "``twistd``"
daemon-launching tool. On some platforms (including Windows) this command is
unable to run a daemon in the background; in that case it behaves in the
same way as "``tahoe run``".
Running Nodes
-------------
"``tahoe stop [NODEDIR]``" will shut down a running node.
No matter what kind of node you created, the correct way to run it is
to use the ``tahoe run`` command. "``tahoe run [NODEDIR]``" will start
a previously-created node in the foreground. This command functions
the same way on all platforms and logs to stdout. If you want to run
the process as a daemon, it is recommended that you use your favourite
daemonization tool.
"``tahoe restart [NODEDIR]``" will stop and then restart a running node. This
is most often used by developers who have just modified the code and want to
start using their changes.
The now-deprecated "``tahoe start [NODEDIR]``" command will launch a
previously-created node. It will launch the node into the background
using ``tahoe daemonize`` (and internal-only command, not for user
use). On some platforms (including Windows) this command is unable to
run a daemon in the background; in that case it behaves in the same
way as "``tahoe run``". ``tahoe start`` also monitors the logs for up
to 5 seconds looking for either a succesful startup message or for
early failure messages and produces an appropriate exit code. You are
encouraged to use ``tahoe run`` along with your favourite
daemonization tool instead of this. ``tahoe start`` is maintained for
backwards compatibility of users already using it; new scripts should
depend on ``tahoe run``.
"``tahoe stop [NODEDIR]``" will shut down a running node. "``tahoe
restart [NODEDIR]``" will stop and then restart a running
node. Similar to above, you should use ``tahoe run`` instead alongside
your favourite daemonization tool.
File Store Manipulation

30
src/allmydata/scripts/runner.py
View File

@ -6,8 +6,9 @@ from twisted.python import usage
from twisted.internet import defer, task, threads
from allmydata.scripts.common import get_default_nodedir
from allmydata.scripts import debug, create_node, startstop_node, cli, \
stats_gatherer, admin, magic_folder_cli, tahoe_invite
from allmydata.scripts import debug, create_node, cli, \
stats_gatherer, admin, magic_folder_cli, tahoe_daemonize, tahoe_start, \
tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite
from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding
def GROUP(s):
@ -29,6 +30,17 @@ NODEDIR_HELP = ("Specify which Tahoe node directory should be used. The "
if _default_nodedir:
NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
# XXX all this 'dispatch' stuff needs to be unified + fixed up
_control_node_dispatch = {
"daemonize": tahoe_daemonize.daemonize,
"start": tahoe_start.start,
"run": tahoe_run.run,
"stop": tahoe_stop.stop,
"restart": tahoe_restart.restart,
}
class Options(usage.Options):
# unit tests can override these to point at StringIO instances
stdin = sys.stdin
@ -41,7 +53,13 @@ class Options(usage.Options):
+ stats_gatherer.subCommands
+ admin.subCommands
+ GROUP("Controlling a node")
+ startstop_node.subCommands
+ [
["daemonize", None, tahoe_daemonize.DaemonizeOptions, "run a node in the background"],
["start", None, tahoe_start.StartOptions, "start a node in the background and confirm it started"],
["run", None, tahoe_run.RunOptions, "run a node without daemonizing"],
["stop", None, tahoe_stop.StopOptions, "stop a node"],
["restart", None, tahoe_restart.RestartOptions, "restart a node"],
]
+ GROUP("Debugging")
+ debug.subCommands
+ GROUP("Using the file store")
@ -104,7 +122,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout):
config = Options()
try:
parse_options(argv, config=config)
except usage.error, e:
except usage.error as e:
c = config
while hasattr(c, 'subOptions'):
c = c.subOptions
@ -129,8 +147,8 @@ def dispatch(config,
if command in create_dispatch:
f = create_dispatch[command]
elif command in startstop_node.dispatch:
f = startstop_node.dispatch[command]
elif command in _control_node_dispatch:
f = _control_node_dispatch[command]
elif command in debug.dispatch:
f = debug.dispatch[command]
elif command in admin.dispatch:

276
src/allmydata/scripts/startstop_node.py
View File

@ -1,276 +0,0 @@
import os, sys, signal, time
from allmydata.scripts.common import BasedirOptions
from twisted.scripts import twistd
from twisted.python import usage
from allmydata.scripts.default_nodedir import _default_nodedir
from allmydata.util import fileutil
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
class StartOptions(BasedirOptions):
subcommand_name = "start"
optParameters = [
("basedir", "C", None,
"Specify which Tahoe base directory should be used."
" This has the same effect as the global --node-directory option."
" [default: %s]" % quote_local_unicode_path(_default_nodedir)),
]
def parseArgs(self, basedir=None, *twistd_args):
# This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon'
# looks like an option to the tahoe subcommand, not to twistd. So you
# can either use 'tahoe start' or 'tahoe start NODEDIR
# --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start
# --TWISTD-OPTIONS' also isn't allowed, unfortunately.
BasedirOptions.parseArgs(self, basedir)
self.twistd_args = twistd_args
def getSynopsis(self):
return ("Usage: %s [global-options] %s [options]"
" [NODEDIR [twistd-options]]"
% (self.command_name, self.subcommand_name))
def getUsage(self, width=None):
t = BasedirOptions.getUsage(self, width) + "\n"
twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0]
t += twistd_options.replace("Options:", "twistd-options:", 1)
t += """
Note that if any twistd-options are used, NODEDIR must be specified explicitly
(not by default or using -C/--basedir or -d/--node-directory), and followed by
the twistd-options.
"""
return t
class StopOptions(BasedirOptions):
def parseArgs(self, basedir=None):
BasedirOptions.parseArgs(self, basedir)
def getSynopsis(self):
return ("Usage: %s [global-options] stop [options] [NODEDIR]"
% (self.command_name,))
class RestartOptions(StartOptions):
subcommand_name = "restart"
class RunOptions(StartOptions):
subcommand_name = "run"
class MyTwistdConfig(twistd.ServerOptions):
subCommands = [("StartTahoeNode", None, usage.Options, "node")]
class StartTahoeNodePlugin:
tapname = "tahoenode"
def __init__(self, nodetype, basedir):
self.nodetype = nodetype
self.basedir = basedir
def makeService(self, so):
# delay this import as late as possible, to allow twistd's code to
# accept --reactor= selection. N.B.: this can't actually work until
# this file, and all the __init__.py files above it, also respect the
# prohibition on importing anything that transitively imports
# twisted.internet.reactor . That will take a lot of work.
if self.nodetype == "client":
from allmydata.client import Client
return Client(self.basedir)
if self.nodetype == "introducer":
from allmydata.introducer.server import IntroducerNode
return IntroducerNode(self.basedir)
if self.nodetype == "key-generator":
raise ValueError("key-generator support removed, see #2783")
if self.nodetype == "stats-gatherer":
from allmydata.stats import StatsGathererService
return StatsGathererService(verbose=True)
raise ValueError("unknown nodetype %s" % self.nodetype)
def identify_node_type(basedir):
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
return None
for t in ("client", "introducer", "key-generator", "stats-gatherer"):
if t in tac:
return t
return None
def start(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STARTING", quoted_basedir
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in ("client", "introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("StartTahoeNode") # point at our StartTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >>err, config
print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue)
return 1
twistd_config.loadedPlugins = {"StartTahoeNode": StartTahoeNodePlugin(nodetype, basedir)}
# On Unix-like platforms:
# Unless --nodaemon was provided, the twistd.runApp() below spawns off a
# child process, and the parent calls os._exit(0), so there's no way for
# us to get control afterwards, even with 'except SystemExit'. If
# application setup fails (e.g. ImportError), runApp() will raise an
# exception.
#
# So if we wanted to do anything with the running child, we'd have two
# options:
#
# * fork first, and have our child wait for the runApp() child to get
# running. (note: just fork(). This is easier than fork+exec, since we
# don't have to get PATH and PYTHONPATH set up, since we're not
# starting a *different* process, just cloning a new instance of the
# current process)
# * or have the user run a separate command some time after this one
# exits.
#
# For Tahoe, we don't need to do anything with the child, so we can just
# let it exit.
#
# On Windows:
# twistd does not fork; it just runs in the current process whether or not
# --nodaemon is specified. (As on Unix, --nodaemon does have the side effect
# of causing us to log to stdout/stderr.)
if "--nodaemon" in twistd_args or sys.platform == "win32":
verb = "running"
else:
verb = "starting"
print >>out, "%s node in %s" % (verb, quoted_basedir)
twistd.runApp(twistd_config)
# we should only reach here if --nodaemon or equivalent was used
return 0
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STOPPING", quoted_basedir
pidfile = os.path.join(basedir, u"twistd.pid")
if not os.path.exists(pidfile):
print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
with open(pidfile, "r") as f:
pid = f.read()
try:
pid = int(pid)
except ValueError:
# The error message below mimics a Twisted error message, which is
# displayed when starting a node with an invalid pidfile.
print >>err, "Pidfile %s contains non-numeric value" % pidfile
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return 2
else:
raise
try:
os.remove(pidfile)
except EnvironmentError:
pass
start = time.time()
time.sleep(0.1)
wait = 40
first_time = True
while True:
# poll once per second until we see the process is no longer running
try:
os.kill(pid, 0)
except OSError:
print >>out, "process %d is dead" % pid
return
wait -= 1
if wait < 0:
if first_time:
print >>err, ("It looks like pid %d is still running "
"after %d seconds" % (pid,
(time.time() - start)))
print >>err, "I will keep watching it until you interrupt me."
wait = 10
first_time = False
else:
print >>err, "pid %d still running after %d seconds" % \
(pid, (time.time() - start))
wait = 10
time.sleep(1)
# we define rc=1 to mean "I think something is still running, sorry"
return 1
def restart(config):
stderr = config.stderr
rc = stop(config)
if rc == 2:
print >>stderr, "ignoring couldn't-stop"
rc = 0
if rc:
print >>stderr, "not restarting"
return rc
return start(config)
def run(config):
config.twistd_args = config.twistd_args + ("--nodaemon",)
# Previously we would do the equivalent of adding ("--logfile",
# "tahoesvc.log"), but that redirects stdout/stderr which is often
# unhelpful, and the user can add that option explicitly if they want.
return start(config)
subCommands = [
["start", None, StartOptions, "Start a node (of any type)."],
["stop", None, StopOptions, "Stop a node."],
["restart", None, RestartOptions, "Restart a node."],
["run", None, RunOptions, "Run a node synchronously."],
]
dispatch = {
"start": start,
"stop": stop,
"restart": restart,
"run": run,
}

192
src/allmydata/scripts/tahoe_daemonize.py Normal file
View File

@ -0,0 +1,192 @@
import os, sys
from allmydata.scripts.common import BasedirOptions
from twisted.scripts import twistd
from twisted.python import usage
from twisted.python.reflect import namedAny
from allmydata.scripts.default_nodedir import _default_nodedir
from allmydata.util import fileutil
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
from twisted.application.service import Service
def identify_node_type(basedir):
"""
:return unicode: None or one of: 'client', 'introducer',
'key-generator' or 'stats-gatherer'
"""
tac = u''
try:
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = fn
break
except OSError:
return None
for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"):
if t in tac:
return t
return None
class DaemonizeOptions(BasedirOptions):
subcommand_name = "start"
optParameters = [
("basedir", "C", None,
"Specify which Tahoe base directory should be used."
" This has the same effect as the global --node-directory option."
" [default: %s]" % quote_local_unicode_path(_default_nodedir)),
]
def parseArgs(self, basedir=None, *twistd_args):
# This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon'
# looks like an option to the tahoe subcommand, not to twistd. So you
# can either use 'tahoe start' or 'tahoe start NODEDIR
# --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start
# --TWISTD-OPTIONS' also isn't allowed, unfortunately.
BasedirOptions.parseArgs(self, basedir)
self.twistd_args = twistd_args
def getSynopsis(self):
return ("Usage: %s [global-options] %s [options]"
" [NODEDIR [twistd-options]]"
% (self.command_name, self.subcommand_name))
def getUsage(self, width=None):
t = BasedirOptions.getUsage(self, width) + "\n"
twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0]
t += twistd_options.replace("Options:", "twistd-options:", 1)
t += """
Note that if any twistd-options are used, NODEDIR must be specified explicitly
(not by default or using -C/--basedir or -d/--node-directory), and followed by
the twistd-options.
"""
return t
class MyTwistdConfig(twistd.ServerOptions):
subCommands = [("DaemonizeTahoeNode", None, usage.Options, "node")]
class DaemonizeTheRealService(Service):
def __init__(self, nodetype, basedir, options):
self.nodetype = nodetype
self.basedir = basedir
def startService(self):
def key_generator_removed():
raise ValueError("key-generator support removed, see #2783")
def start():
node_to_instance = {
u"client": lambda: namedAny("allmydata.client.Client")(self.basedir),
u"introducer": lambda: namedAny("allmydata.introducer.server.IntroducerNode")(self.basedir),
u"stats-gatherer": lambda: namedAny("allmydata.stats.StatsGathererService")(self.basedir, verbose=True),
u"key-generator": key_generator_removed,
}
try:
service_factory = node_to_instance[self.nodetype]
except KeyError:
raise ValueError("unknown nodetype %s" % self.nodetype)
srv = service_factory()
srv.setServiceParent(self.parent)
from twisted.internet import reactor
reactor.callWhenRunning(start)
class DaemonizeTahoeNodePlugin(object):
tapname = "tahoenode"
def __init__(self, nodetype, basedir):
self.nodetype = nodetype
self.basedir = basedir
def makeService(self, so):
return DaemonizeTheRealService(self.nodetype, self.basedir, so)
def daemonize(config):
"""
Runs the 'tahoe daemonize' command.
Sets up the IService instance corresponding to the type of node
that's starting and uses Twisted's twistd runner to disconnect our
process from the terminal.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "daemonizing in {}".format(quoted_basedir)
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >>err, config
print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue)
return 1
twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)}
# On Unix-like platforms:
# Unless --nodaemon was provided, the twistd.runApp() below spawns off a
# child process, and the parent calls os._exit(0), so there's no way for
# us to get control afterwards, even with 'except SystemExit'. If
# application setup fails (e.g. ImportError), runApp() will raise an
# exception.
#
# So if we wanted to do anything with the running child, we'd have two
# options:
#
# * fork first, and have our child wait for the runApp() child to get
# running. (note: just fork(). This is easier than fork+exec, since we
# don't have to get PATH and PYTHONPATH set up, since we're not
# starting a *different* process, just cloning a new instance of the
# current process)
# * or have the user run a separate command some time after this one
# exits.
#
# For Tahoe, we don't need to do anything with the child, so we can just
# let it exit.
#
# On Windows:
# twistd does not fork; it just runs in the current process whether or not
# --nodaemon is specified. (As on Unix, --nodaemon does have the side effect
# of causing us to log to stdout/stderr.)
if "--nodaemon" in twistd_args or sys.platform == "win32":
verb = "running"
else:
verb = "starting"
print >>out, "%s node in %s" % (verb, quoted_basedir)
twistd.runApp(twistd_config)
# we should only reach here if --nodaemon or equivalent was used
return 0

18
src/allmydata/scripts/tahoe_restart.py Normal file
View File

@ -0,0 +1,18 @@
from .tahoe_start import StartOptions, start
from .tahoe_stop import stop, COULD_NOT_STOP
class RestartOptions(StartOptions):
subcommand_name = "restart"
def restart(config):
stderr = config.stderr
rc = stop(config)
if rc == COULD_NOT_STOP:
print >>stderr, "ignoring couldn't-stop"
rc = 0
if rc:
print >>stderr, "not restarting"
return rc
return start(config)

10
src/allmydata/scripts/tahoe_run.py Normal file
View File

@ -0,0 +1,10 @@
from .tahoe_daemonize import daemonize, DaemonizeOptions
class RunOptions(DaemonizeOptions):
subcommand_name = "run"
def run(config):
config.twistd_args = config.twistd_args + ("--nodaemon",)
return daemonize(config)

139
src/allmydata/scripts/tahoe_start.py Normal file
View File

@ -0,0 +1,139 @@
import os
import io
import sys
import time
import subprocess
from os.path import join, exists
from allmydata.scripts.common import BasedirOptions
from allmydata.scripts.default_nodedir import _default_nodedir
from allmydata.util.encodingutil import quote_local_unicode_path
from .tahoe_daemonize import MyTwistdConfig, identify_node_type
class StartOptions(BasedirOptions):
subcommand_name = "start"
optParameters = [
("basedir", "C", None,
"Specify which Tahoe base directory should be used."
" This has the same effect as the global --node-directory option."
" [default: %s]" % quote_local_unicode_path(_default_nodedir)),
]
def parseArgs(self, basedir=None, *twistd_args):
# This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon'
# looks like an option to the tahoe subcommand, not to twistd. So you
# can either use 'tahoe start' or 'tahoe start NODEDIR
# --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start
# --TWISTD-OPTIONS' also isn't allowed, unfortunately.
BasedirOptions.parseArgs(self, basedir)
self.twistd_args = twistd_args
def getSynopsis(self):
return ("Usage: %s [global-options] %s [options]"
" [NODEDIR [twistd-options]]"
% (self.command_name, self.subcommand_name))
def getUsage(self, width=None):
t = BasedirOptions.getUsage(self, width) + "\n"
twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0]
t += twistd_options.replace("Options:", "twistd-options:", 1)
t += """
Note that if any twistd-options are used, NODEDIR must be specified explicitly
(not by default or using -C/--basedir or -d/--node-directory), and followed by
the twistd-options.
"""
return t
def start(config):
"""
Start a tahoe node (daemonize it and confirm startup)
We run 'tahoe daemonize' with all the options given to 'tahoe
start' and then watch the log files for the correct text to appear
(e.g. "introducer started"). If that doesn't happen within a few
seconds, an error is printed along with all collected logs.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STARTING", quoted_basedir
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# "tahoe start" attempts to monitor the logs for successful
# startup -- but we can't always do that.
can_monitor_logs = False
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
can_monitor_logs = True
if "--help" in config.twistd_args:
return 0
if not can_monitor_logs:
print >>out, "Custom logging options; can't monitor logs for proper startup messages"
return 1
# before we spawn tahoe, we check if "the log file" exists or not,
# and if so remember how big it is -- essentially, we're doing
# "tail -f" to see what "this" incarnation of "tahoe daemonize"
# spews forth.
starting_offset = 0
log_fname = join(basedir, 'logs', 'twistd.log')
if exists(log_fname):
with open(log_fname, 'r') as f:
f.seek(0, 2)
starting_offset = f.tell()
# spawn tahoe. Note that since this daemonizes, it should return
# "pretty fast" and with a zero return-code, or else something
# Very Bad has happened.
try:
args = [sys.executable]
for i, arg in enumerate(sys.argv):
if arg in ['start', 'restart']:
args.append('daemonize')
else:
args.append(arg)
subprocess.check_call(args)
except subprocess.CalledProcessError as e:
return e.returncode
# now, we have to determine if tahoe has actually started up
# successfully or not. so, we start sucking up log files and
# looking for "the magic string", which depends on the node type.
magic_string = u'{} running'.format(nodetype)
with io.open(log_fname, 'r') as f:
f.seek(starting_offset)
collected = u''
start = time.time()
while time.time() - start < 5:
collected += f.read()
if magic_string in collected:
if not config.parent['quiet']:
print >>out, "Node has started successfully"
return 0
time.sleep(0.1)
print >>out, "Something has gone wrong starting the node."
print >>out, "Logs are available in '{}'".format(log_fname)
print >>out, "Collected for this run:"
print >>out, collected
return 1

88
src/allmydata/scripts/tahoe_stop.py Normal file
View File

@ -0,0 +1,88 @@
import os
import time
import signal
from allmydata.scripts.common import BasedirOptions
from allmydata.util.encodingutil import quote_local_unicode_path
COULD_NOT_STOP = 2
class StopOptions(BasedirOptions):
def parseArgs(self, basedir=None):
BasedirOptions.parseArgs(self, basedir)
def getSynopsis(self):
return ("Usage: %s [global-options] stop [options] [NODEDIR]"
% (self.command_name,))
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STOPPING", quoted_basedir
pidfile = os.path.join(basedir, u"twistd.pid")
if not os.path.exists(pidfile):
print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return COULD_NOT_STOP
with open(pidfile, "r") as f:
pid = f.read()
try:
pid = int(pid)
except ValueError:
# The error message below mimics a Twisted error message, which is
# displayed when starting a node with an invalid pidfile.
print >>err, "Pidfile %s contains non-numeric value" % pidfile
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return COULD_NOT_STOP
else:
raise
try:
os.remove(pidfile)
except EnvironmentError:
pass
start = time.time()
time.sleep(0.1)
wait = 40
first_time = True
while True:
# poll once per second until we see the process is no longer running
try:
os.kill(pid, 0)
except OSError:
print >>out, "process %d is dead" % pid
return
wait -= 1
if wait < 0:
if first_time:
print >>err, ("It looks like pid %d is still running "
"after %d seconds" % (pid,
(time.time() - start)))
print >>err, "I will keep watching it until you interrupt me."
wait = 10
first_time = False
else:
print >>err, "pid %d still running after %d seconds" % \
(pid, (time.time() - start))
wait = 10
time.sleep(1)
# we define rc=1 to mean "I think something is still running, sorry"
return 1

22
src/allmydata/test/cli/test_cli.py
View File

@ -20,12 +20,14 @@ import allmydata.scripts.common_http
from pycryptopp.publickey import ed25519
# Test that the scripts can be imported.
from allmydata.scripts import create_node, debug, startstop_node, \
from allmydata.scripts import create_node, debug, tahoe_start, tahoe_restart, \
tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls, \
tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen
_hush_pyflakes = [create_node, debug, startstop_node,
tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen, \
tahoe_stop, tahoe_daemonize, tahoe_run
_hush_pyflakes = [create_node, debug, tahoe_start, tahoe_restart, tahoe_stop,
tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls,
tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen]
tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen,
tahoe_daemonize, tahoe_run]
from allmydata.scripts import common
from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \
@ -630,19 +632,19 @@ class Help(unittest.TestCase):
self.failUnlessIn("[options]", help)
def test_start(self):
help = str(startstop_node.StartOptions())
help = str(tahoe_start.StartOptions())
self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help)
def test_stop(self):
help = str(startstop_node.StopOptions())
help = str(tahoe_stop.StopOptions())
self.failUnlessIn("[options] [NODEDIR]", help)
def test_restart(self):
help = str(startstop_node.RestartOptions())
help = str(tahoe_restart.RestartOptions())
self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help)
def test_run(self):
help = str(startstop_node.RunOptions())
help = str(tahoe_run.RunOptions())
self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help)
def test_create_client(self):
@ -1303,11 +1305,11 @@ class Stop(unittest.TestCase):
basedir.makedirs()
basedir.child(u"twistd.pid").setContent(b"foo")
config = startstop_node.StopOptions()
config = tahoe_stop.StopOptions()
config.stdout = StringIO()
config.stderr = StringIO()
config['basedir'] = basedir.path
result_code = startstop_node.stop(config)
result_code = tahoe_stop.stop(config)
self.assertEqual(2, result_code)
self.assertIn("contains non-numeric value", config.stderr.getvalue())

177
src/allmydata/test/cli/test_daemonize.py Normal file
View File

@ -0,0 +1,177 @@
import os
from os.path import dirname, join
from mock import patch, Mock
from StringIO import StringIO
from sys import getfilesystemencoding
from twisted.trial import unittest
from allmydata.scripts import runner
from allmydata.scripts.tahoe_daemonize import identify_node_type
from allmydata.scripts.tahoe_daemonize import DaemonizeTahoeNodePlugin
from allmydata.scripts.tahoe_daemonize import DaemonizeOptions
class Util(unittest.TestCase):
def test_node_type_nothing(self):
tmpdir = self.mktemp()
base = dirname(tmpdir).decode(getfilesystemencoding())
t = identify_node_type(base)
self.assertIs(None, t)
def test_node_type_introducer(self):
tmpdir = self.mktemp()
base = dirname(tmpdir).decode(getfilesystemencoding())
with open(join(dirname(tmpdir), 'introducer.tac'), 'w') as f:
f.write("test placeholder")
t = identify_node_type(base)
self.assertEqual(u"introducer", t)
def test_daemonize(self):
tmpdir = self.mktemp()
plug = DaemonizeTahoeNodePlugin('client', tmpdir)
with patch('twisted.internet.reactor') as r:
def call(fn, *args, **kw):
fn()
r.callWhenRunning = call
service = plug.makeService(None)
service.parent = Mock()
service.startService()
self.assertTrue(service is not None)
def test_daemonize_no_keygen(self):
tmpdir = self.mktemp()
plug = DaemonizeTahoeNodePlugin('key-generator', tmpdir)
with patch('twisted.internet.reactor') as r:
def call(fn, *args, **kw):
fn()
r.callWhenRunning = call
service = plug.makeService(None)
service.parent = Mock()
with self.assertRaises(ValueError) as ctx:
service.startService()
self.assertIn(
"key-generator support removed",
str(ctx.exception)
)
def test_daemonize_unknown_nodetype(self):
tmpdir = self.mktemp()
plug = DaemonizeTahoeNodePlugin('an-unknown-service', tmpdir)
with patch('twisted.internet.reactor') as r:
def call(fn, *args, **kw):
fn()
r.callWhenRunning = call
service = plug.makeService(None)
service.parent = Mock()
with self.assertRaises(ValueError) as ctx:
service.startService()
self.assertIn(
"unknown nodetype",
str(ctx.exception)
)
def test_daemonize_options(self):
parent = runner.Options()
opts = DaemonizeOptions()
opts.parent = parent
opts.parseArgs()
# just gratuitous coverage, ensureing we don't blow up on
# these methods.
opts.getSynopsis()
opts.getUsage()
class RunDaemonizeTests(unittest.TestCase):
def setUp(self):
# no test should change our working directory
self._working = os.path.abspath('.')
d = super(RunDaemonizeTests, self).setUp()
self._reactor = patch('twisted.internet.reactor')
self._twistd = patch('allmydata.scripts.tahoe_daemonize.twistd')
self.node_dir = self.mktemp()
os.mkdir(self.node_dir)
for cm in [self._reactor, self._twistd]:
cm.__enter__()
return d
def tearDown(self):
d = super(RunDaemonizeTests, self).tearDown()
for cm in [self._reactor, self._twistd]:
cm.__exit__(None, None, None)
# Note: if you raise an exception (e.g. via self.assertEqual
# or raise RuntimeError) it is apparently just ignored and the
# test passes anyway...
if self._working != os.path.abspath('.'):
print("WARNING: a test just changed the working dir; putting it back")
os.chdir(self._working)
return d
def _placeholder_nodetype(self, nodetype):
fname = join(self.node_dir, '{}.tac'.format(nodetype))
with open(fname, 'w') as f:
f.write("test placeholder")
def test_daemonize_defaults(self):
self._placeholder_nodetype('introducer')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't much around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'daemonize',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
runner.dispatch(config, i, o, e)
self.assertEqual(0, exit_code[0])
def test_daemonize_wrong_nodetype(self):
self._placeholder_nodetype('invalid')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't much around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'daemonize',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
runner.dispatch(config, i, o, e)
self.assertEqual(0, exit_code[0])
def test_daemonize_run(self):
self._placeholder_nodetype('client')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't much around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'daemonize',
])
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
from allmydata.scripts.tahoe_daemonize import daemonize
daemonize(config)

209
src/allmydata/test/cli/test_start.py Normal file
View File

@ -0,0 +1,209 @@
import os
import shutil
import subprocess
from os.path import join
from mock import patch
from StringIO import StringIO
from twisted.trial import unittest
from allmydata.scripts import runner
#@patch('twisted.internet.reactor')
@patch('allmydata.scripts.tahoe_start.subprocess')
class RunStartTests(unittest.TestCase):
def setUp(self):
d = super(RunStartTests, self).setUp()
self.node_dir = self.mktemp()
os.mkdir(self.node_dir)
return d
def _placeholder_nodetype(self, nodetype):
fname = join(self.node_dir, '{}.tac'.format(nodetype))
with open(fname, 'w') as f:
f.write("test placeholder")
def _pid_file(self, pid):
fname = join(self.node_dir, 'twistd.pid')
with open(fname, 'w') as f:
f.write(u"{}\n".format(pid))
def _logs(self, logs):
os.mkdir(join(self.node_dir, 'logs'))
fname = join(self.node_dir, 'logs', 'twistd.log')
with open(fname, 'w') as f:
f.write(logs)
def test_start_defaults(self, _subprocess):
self._placeholder_nodetype('client')
self._pid_file(1234)
self._logs('one log\ntwo log\nred log\nblue log\n')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
])
i, o, e = StringIO(), StringIO(), StringIO()
try:
with patch('allmydata.scripts.tahoe_start.os'):
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
def launch(*args, **kw):
with open(join(self.node_dir, 'logs', 'twistd.log'), 'a') as f:
f.write('client running\n') # "the magic"
_subprocess.check_call = launch
runner.dispatch(config, i, o, e)
except Exception:
pass
self.assertEqual([0], exit_code)
self.assertTrue('Node has started' in o.getvalue())
def test_start_fails(self, _subprocess):
self._placeholder_nodetype('client')
self._logs('existing log line\n')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.tahoe_start.time') as t:
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
thetime = [0]
def _time():
thetime[0] += 0.1
return thetime[0]
t.time = _time
def launch(*args, **kw):
with open(join(self.node_dir, 'logs', 'twistd.log'), 'a') as f:
f.write('a new log line\n')
_subprocess.check_call = launch
runner.dispatch(config, i, o, e)
# should print out the collected logs and an error-code
self.assertTrue("a new log line" in o.getvalue())
self.assertEqual([1], exit_code)
def test_start_subprocess_fails(self, _subprocess):
self._placeholder_nodetype('client')
self._logs('existing log line\n')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.tahoe_start.time'):
with patch('allmydata.scripts.runner.sys') as s:
# undo patch for the exception-class
_subprocess.CalledProcessError = subprocess.CalledProcessError
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
def launch(*args, **kw):
raise subprocess.CalledProcessError(42, "tahoe")
_subprocess.check_call = launch
runner.dispatch(config, i, o, e)
# should get our "odd" error-code
self.assertEqual([42], exit_code)
def test_start_help(self, _subprocess):
self._placeholder_nodetype('client')
std = StringIO()
with patch('sys.stdout') as stdo:
stdo.write = std.write
try:
runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
'--help',
], stdout=std)
self.fail("Should get exit")
except SystemExit as e:
print(e)
self.assertIn(
"Usage:",
std.getvalue()
)
def test_start_unknown_node_type(self, _subprocess):
self._placeholder_nodetype('bogus')
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
runner.dispatch(config, i, o, e)
# should print out the collected logs and an error-code
self.assertIn(
"is not a recognizable node directory",
e.getvalue()
)
self.assertEqual([1], exit_code)
def test_start_nodedir_not_dir(self, _subprocess):
shutil.rmtree(self.node_dir)
assert not os.path.isdir(self.node_dir)
config = runner.parse_or_exit_with_explanation([
# have to do this so the tests don't muck around in
# ~/.tahoe (the default)
'--node-directory', self.node_dir,
'start',
])
i, o, e = StringIO(), StringIO(), StringIO()
with patch('allmydata.scripts.runner.sys') as s:
exit_code = [None]
def _exit(code):
exit_code[0] = code
s.exit = _exit
runner.dispatch(config, i, o, e)
# should print out the collected logs and an error-code
self.assertIn(
"does not look like a directory at all",
e.getvalue()
)
self.assertEqual([1], exit_code)

14
src/allmydata/test/test_upload.py
View File

@ -860,6 +860,20 @@ class FakeServerTracker:
class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
ShouldFailMixin):
def setUp(self):
d = super(EncodingParameters, self).setUp()
self._curdir = os.path.abspath(os.path.curdir)
return d
def tearDown(self):
d = super(EncodingParameters, self).tearDown()
self.assertEqual(
os.path.abspath(os.path.curdir),
self._curdir,
)
return d
def find_all_shares(self, unused=None):
"""Locate shares on disk. Returns a dict that maps
server to set of sharenums.