diff --git a/docs/frontends/CLI.rst b/docs/frontends/CLI.rst index 70dc5ee90..9b6343c58 100644 --- a/docs/frontends/CLI.rst +++ b/docs/frontends/CLI.rst @@ -83,12 +83,13 @@ the command line. Node Management =============== -"``tahoe create-node [NODEDIR]``" is the basic make-a-new-node command. It -creates a new directory and populates it with files that will allow the -"``tahoe start``" command to use it later on. This command creates nodes that -have client functionality (upload/download files), web API services -(controlled by the '[node]web.port' configuration), and storage services -(unless ``--no-storage`` is specified). +"``tahoe create-node [NODEDIR]``" is the basic make-a-new-node +command. It creates a new directory and populates it with files that +will allow the "``tahoe start``" and related commands to use it later +on. ``tahoe create-node`` creates nodes that have client functionality +(upload/download files), web API services (controlled by the +'[node]web.port' configuration), and storage services (unless +``--no-storage`` is specified). NODEDIR defaults to ``~/.tahoe/`` , and newly-created nodes default to publishing a web server on port 3456 (limited to the loopback interface, at @@ -105,19 +106,34 @@ This node provides introduction services and nothing else. When started, this node will produce a ``private/introducer.furl`` file, which should be published to all clients. -"``tahoe run [NODEDIR]``" will start a previously-created node in the foreground. -"``tahoe start [NODEDIR]``" will launch a previously-created node. It will -launch the node into the background, using the standard Twisted "``twistd``" -daemon-launching tool. On some platforms (including Windows) this command is -unable to run a daemon in the background; in that case it behaves in the -same way as "``tahoe run``". +Running Nodes +------------- -"``tahoe stop [NODEDIR]``" will shut down a running node. +No matter what kind of node you created, the correct way to run it is +to use the ``tahoe run`` command. "``tahoe run [NODEDIR]``" will start +a previously-created node in the foreground. This command functions +the same way on all platforms and logs to stdout. If you want to run +the process as a daemon, it is recommended that you use your favourite +daemonization tool. -"``tahoe restart [NODEDIR]``" will stop and then restart a running node. This -is most often used by developers who have just modified the code and want to -start using their changes. +The now-deprecated "``tahoe start [NODEDIR]``" command will launch a +previously-created node. It will launch the node into the background +using ``tahoe daemonize`` (and internal-only command, not for user +use). On some platforms (including Windows) this command is unable to +run a daemon in the background; in that case it behaves in the same +way as "``tahoe run``". ``tahoe start`` also monitors the logs for up +to 5 seconds looking for either a succesful startup message or for +early failure messages and produces an appropriate exit code. You are +encouraged to use ``tahoe run`` along with your favourite +daemonization tool instead of this. ``tahoe start`` is maintained for +backwards compatibility of users already using it; new scripts should +depend on ``tahoe run``. + +"``tahoe stop [NODEDIR]``" will shut down a running node. "``tahoe +restart [NODEDIR]``" will stop and then restart a running +node. Similar to above, you should use ``tahoe run`` instead alongside +your favourite daemonization tool. File Store Manipulation diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index ffe6f9fc3..d616cf199 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -6,8 +6,9 @@ from twisted.python import usage from twisted.internet import defer, task, threads from allmydata.scripts.common import get_default_nodedir -from allmydata.scripts import debug, create_node, startstop_node, cli, \ - stats_gatherer, admin, magic_folder_cli, tahoe_invite +from allmydata.scripts import debug, create_node, cli, \ + stats_gatherer, admin, magic_folder_cli, tahoe_daemonize, tahoe_start, \ + tahoe_stop, tahoe_restart, tahoe_run, tahoe_invite from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding def GROUP(s): @@ -29,6 +30,17 @@ NODEDIR_HELP = ("Specify which Tahoe node directory should be used. The " if _default_nodedir: NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]" + +# XXX all this 'dispatch' stuff needs to be unified + fixed up +_control_node_dispatch = { + "daemonize": tahoe_daemonize.daemonize, + "start": tahoe_start.start, + "run": tahoe_run.run, + "stop": tahoe_stop.stop, + "restart": tahoe_restart.restart, +} + + class Options(usage.Options): # unit tests can override these to point at StringIO instances stdin = sys.stdin @@ -41,7 +53,13 @@ class Options(usage.Options): + stats_gatherer.subCommands + admin.subCommands + GROUP("Controlling a node") - + startstop_node.subCommands + + [ + ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "run a node in the background"], + ["start", None, tahoe_start.StartOptions, "start a node in the background and confirm it started"], + ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], + ["stop", None, tahoe_stop.StopOptions, "stop a node"], + ["restart", None, tahoe_restart.RestartOptions, "restart a node"], + ] + GROUP("Debugging") + debug.subCommands + GROUP("Using the file store") @@ -104,7 +122,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout): config = Options() try: parse_options(argv, config=config) - except usage.error, e: + except usage.error as e: c = config while hasattr(c, 'subOptions'): c = c.subOptions @@ -129,8 +147,8 @@ def dispatch(config, if command in create_dispatch: f = create_dispatch[command] - elif command in startstop_node.dispatch: - f = startstop_node.dispatch[command] + elif command in _control_node_dispatch: + f = _control_node_dispatch[command] elif command in debug.dispatch: f = debug.dispatch[command] elif command in admin.dispatch: diff --git a/src/allmydata/scripts/startstop_node.py b/src/allmydata/scripts/startstop_node.py deleted file mode 100644 index 75073a1e8..000000000 --- a/src/allmydata/scripts/startstop_node.py +++ /dev/null @@ -1,276 +0,0 @@ - -import os, sys, signal, time -from allmydata.scripts.common import BasedirOptions -from twisted.scripts import twistd -from twisted.python import usage -from allmydata.scripts.default_nodedir import _default_nodedir -from allmydata.util import fileutil -from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path - - -class StartOptions(BasedirOptions): - subcommand_name = "start" - optParameters = [ - ("basedir", "C", None, - "Specify which Tahoe base directory should be used." - " This has the same effect as the global --node-directory option." - " [default: %s]" % quote_local_unicode_path(_default_nodedir)), - ] - - def parseArgs(self, basedir=None, *twistd_args): - # This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon' - # looks like an option to the tahoe subcommand, not to twistd. So you - # can either use 'tahoe start' or 'tahoe start NODEDIR - # --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start - # --TWISTD-OPTIONS' also isn't allowed, unfortunately. - - BasedirOptions.parseArgs(self, basedir) - self.twistd_args = twistd_args - - def getSynopsis(self): - return ("Usage: %s [global-options] %s [options]" - " [NODEDIR [twistd-options]]" - % (self.command_name, self.subcommand_name)) - - def getUsage(self, width=None): - t = BasedirOptions.getUsage(self, width) + "\n" - twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0] - t += twistd_options.replace("Options:", "twistd-options:", 1) - t += """ - -Note that if any twistd-options are used, NODEDIR must be specified explicitly -(not by default or using -C/--basedir or -d/--node-directory), and followed by -the twistd-options. -""" - return t - -class StopOptions(BasedirOptions): - def parseArgs(self, basedir=None): - BasedirOptions.parseArgs(self, basedir) - - def getSynopsis(self): - return ("Usage: %s [global-options] stop [options] [NODEDIR]" - % (self.command_name,)) - -class RestartOptions(StartOptions): - subcommand_name = "restart" - -class RunOptions(StartOptions): - subcommand_name = "run" - - -class MyTwistdConfig(twistd.ServerOptions): - subCommands = [("StartTahoeNode", None, usage.Options, "node")] - -class StartTahoeNodePlugin: - tapname = "tahoenode" - def __init__(self, nodetype, basedir): - self.nodetype = nodetype - self.basedir = basedir - def makeService(self, so): - # delay this import as late as possible, to allow twistd's code to - # accept --reactor= selection. N.B.: this can't actually work until - # this file, and all the __init__.py files above it, also respect the - # prohibition on importing anything that transitively imports - # twisted.internet.reactor . That will take a lot of work. - if self.nodetype == "client": - from allmydata.client import Client - return Client(self.basedir) - if self.nodetype == "introducer": - from allmydata.introducer.server import IntroducerNode - return IntroducerNode(self.basedir) - if self.nodetype == "key-generator": - raise ValueError("key-generator support removed, see #2783") - if self.nodetype == "stats-gatherer": - from allmydata.stats import StatsGathererService - return StatsGathererService(verbose=True) - raise ValueError("unknown nodetype %s" % self.nodetype) - -def identify_node_type(basedir): - for fn in listdir_unicode(basedir): - if fn.endswith(u".tac"): - tac = str(fn) - break - else: - return None - - for t in ("client", "introducer", "key-generator", "stats-gatherer"): - if t in tac: - return t - return None - -def start(config): - out = config.stdout - err = config.stderr - basedir = config['basedir'] - quoted_basedir = quote_local_unicode_path(basedir) - print >>out, "STARTING", quoted_basedir - if not os.path.isdir(basedir): - print >>err, "%s does not look like a directory at all" % quoted_basedir - return 1 - nodetype = identify_node_type(basedir) - if not nodetype: - print >>err, "%s is not a recognizable node directory" % quoted_basedir - return 1 - # Now prepare to turn into a twistd process. This os.chdir is the point - # of no return. - os.chdir(basedir) - twistd_args = [] - if (nodetype in ("client", "introducer") - and "--nodaemon" not in config.twistd_args - and "--syslog" not in config.twistd_args - and "--logfile" not in config.twistd_args): - fileutil.make_dirs(os.path.join(basedir, u"logs")) - twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")]) - twistd_args.extend(config.twistd_args) - twistd_args.append("StartTahoeNode") # point at our StartTahoeNodePlugin - - twistd_config = MyTwistdConfig() - try: - twistd_config.parseOptions(twistd_args) - except usage.error, ue: - # these arguments were unsuitable for 'twistd' - print >>err, config - print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue) - return 1 - twistd_config.loadedPlugins = {"StartTahoeNode": StartTahoeNodePlugin(nodetype, basedir)} - - # On Unix-like platforms: - # Unless --nodaemon was provided, the twistd.runApp() below spawns off a - # child process, and the parent calls os._exit(0), so there's no way for - # us to get control afterwards, even with 'except SystemExit'. If - # application setup fails (e.g. ImportError), runApp() will raise an - # exception. - # - # So if we wanted to do anything with the running child, we'd have two - # options: - # - # * fork first, and have our child wait for the runApp() child to get - # running. (note: just fork(). This is easier than fork+exec, since we - # don't have to get PATH and PYTHONPATH set up, since we're not - # starting a *different* process, just cloning a new instance of the - # current process) - # * or have the user run a separate command some time after this one - # exits. - # - # For Tahoe, we don't need to do anything with the child, so we can just - # let it exit. - # - # On Windows: - # twistd does not fork; it just runs in the current process whether or not - # --nodaemon is specified. (As on Unix, --nodaemon does have the side effect - # of causing us to log to stdout/stderr.) - - if "--nodaemon" in twistd_args or sys.platform == "win32": - verb = "running" - else: - verb = "starting" - - print >>out, "%s node in %s" % (verb, quoted_basedir) - twistd.runApp(twistd_config) - # we should only reach here if --nodaemon or equivalent was used - return 0 - -def stop(config): - out = config.stdout - err = config.stderr - basedir = config['basedir'] - quoted_basedir = quote_local_unicode_path(basedir) - print >>out, "STOPPING", quoted_basedir - pidfile = os.path.join(basedir, u"twistd.pid") - if not os.path.exists(pidfile): - print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir - # we define rc=2 to mean "nothing is running, but it wasn't me who - # stopped it" - return 2 - with open(pidfile, "r") as f: - pid = f.read() - - try: - pid = int(pid) - except ValueError: - # The error message below mimics a Twisted error message, which is - # displayed when starting a node with an invalid pidfile. - print >>err, "Pidfile %s contains non-numeric value" % pidfile - # we define rc=2 to mean "nothing is running, but it wasn't me who - # stopped it" - return 2 - - # kill it hard (SIGKILL), delete the twistd.pid file, then wait for the - # process itself to go away. If it hasn't gone away after 20 seconds, warn - # the user but keep waiting until they give up. - try: - os.kill(pid, signal.SIGKILL) - except OSError, oserr: - if oserr.errno == 3: - print oserr.strerror - # the process didn't exist, so wipe the pid file - os.remove(pidfile) - return 2 - else: - raise - try: - os.remove(pidfile) - except EnvironmentError: - pass - start = time.time() - time.sleep(0.1) - wait = 40 - first_time = True - while True: - # poll once per second until we see the process is no longer running - try: - os.kill(pid, 0) - except OSError: - print >>out, "process %d is dead" % pid - return - wait -= 1 - if wait < 0: - if first_time: - print >>err, ("It looks like pid %d is still running " - "after %d seconds" % (pid, - (time.time() - start))) - print >>err, "I will keep watching it until you interrupt me." - wait = 10 - first_time = False - else: - print >>err, "pid %d still running after %d seconds" % \ - (pid, (time.time() - start)) - wait = 10 - time.sleep(1) - # we define rc=1 to mean "I think something is still running, sorry" - return 1 - -def restart(config): - stderr = config.stderr - rc = stop(config) - if rc == 2: - print >>stderr, "ignoring couldn't-stop" - rc = 0 - if rc: - print >>stderr, "not restarting" - return rc - return start(config) - -def run(config): - config.twistd_args = config.twistd_args + ("--nodaemon",) - # Previously we would do the equivalent of adding ("--logfile", - # "tahoesvc.log"), but that redirects stdout/stderr which is often - # unhelpful, and the user can add that option explicitly if they want. - - return start(config) - - -subCommands = [ - ["start", None, StartOptions, "Start a node (of any type)."], - ["stop", None, StopOptions, "Stop a node."], - ["restart", None, RestartOptions, "Restart a node."], - ["run", None, RunOptions, "Run a node synchronously."], -] - -dispatch = { - "start": start, - "stop": stop, - "restart": restart, - "run": run, - } diff --git a/src/allmydata/scripts/tahoe_daemonize.py b/src/allmydata/scripts/tahoe_daemonize.py new file mode 100644 index 000000000..da6e02abf --- /dev/null +++ b/src/allmydata/scripts/tahoe_daemonize.py @@ -0,0 +1,192 @@ + +import os, sys +from allmydata.scripts.common import BasedirOptions +from twisted.scripts import twistd +from twisted.python import usage +from twisted.python.reflect import namedAny +from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util import fileutil +from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path +from twisted.application.service import Service + + +def identify_node_type(basedir): + """ + :return unicode: None or one of: 'client', 'introducer', + 'key-generator' or 'stats-gatherer' + """ + tac = u'' + try: + for fn in listdir_unicode(basedir): + if fn.endswith(u".tac"): + tac = fn + break + except OSError: + return None + + for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"): + if t in tac: + return t + return None + + +class DaemonizeOptions(BasedirOptions): + subcommand_name = "start" + optParameters = [ + ("basedir", "C", None, + "Specify which Tahoe base directory should be used." + " This has the same effect as the global --node-directory option." + " [default: %s]" % quote_local_unicode_path(_default_nodedir)), + ] + + def parseArgs(self, basedir=None, *twistd_args): + # This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon' + # looks like an option to the tahoe subcommand, not to twistd. So you + # can either use 'tahoe start' or 'tahoe start NODEDIR + # --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start + # --TWISTD-OPTIONS' also isn't allowed, unfortunately. + + BasedirOptions.parseArgs(self, basedir) + self.twistd_args = twistd_args + + def getSynopsis(self): + return ("Usage: %s [global-options] %s [options]" + " [NODEDIR [twistd-options]]" + % (self.command_name, self.subcommand_name)) + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + "\n" + twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0] + t += twistd_options.replace("Options:", "twistd-options:", 1) + t += """ + +Note that if any twistd-options are used, NODEDIR must be specified explicitly +(not by default or using -C/--basedir or -d/--node-directory), and followed by +the twistd-options. +""" + return t + + +class MyTwistdConfig(twistd.ServerOptions): + subCommands = [("DaemonizeTahoeNode", None, usage.Options, "node")] + + +class DaemonizeTheRealService(Service): + + def __init__(self, nodetype, basedir, options): + self.nodetype = nodetype + self.basedir = basedir + + def startService(self): + + def key_generator_removed(): + raise ValueError("key-generator support removed, see #2783") + + def start(): + node_to_instance = { + u"client": lambda: namedAny("allmydata.client.Client")(self.basedir), + u"introducer": lambda: namedAny("allmydata.introducer.server.IntroducerNode")(self.basedir), + u"stats-gatherer": lambda: namedAny("allmydata.stats.StatsGathererService")(self.basedir, verbose=True), + u"key-generator": key_generator_removed, + } + + try: + service_factory = node_to_instance[self.nodetype] + except KeyError: + raise ValueError("unknown nodetype %s" % self.nodetype) + + srv = service_factory() + srv.setServiceParent(self.parent) + + from twisted.internet import reactor + reactor.callWhenRunning(start) + + +class DaemonizeTahoeNodePlugin(object): + tapname = "tahoenode" + def __init__(self, nodetype, basedir): + self.nodetype = nodetype + self.basedir = basedir + + def makeService(self, so): + return DaemonizeTheRealService(self.nodetype, self.basedir, so) + + +def daemonize(config): + """ + Runs the 'tahoe daemonize' command. + + Sets up the IService instance corresponding to the type of node + that's starting and uses Twisted's twistd runner to disconnect our + process from the terminal. + """ + out = config.stdout + err = config.stderr + basedir = config['basedir'] + quoted_basedir = quote_local_unicode_path(basedir) + print >>out, "daemonizing in {}".format(quoted_basedir) + if not os.path.isdir(basedir): + print >>err, "%s does not look like a directory at all" % quoted_basedir + return 1 + nodetype = identify_node_type(basedir) + if not nodetype: + print >>err, "%s is not a recognizable node directory" % quoted_basedir + return 1 + # Now prepare to turn into a twistd process. This os.chdir is the point + # of no return. + os.chdir(basedir) + twistd_args = [] + if (nodetype in (u"client", u"introducer") + and "--nodaemon" not in config.twistd_args + and "--syslog" not in config.twistd_args + and "--logfile" not in config.twistd_args): + fileutil.make_dirs(os.path.join(basedir, u"logs")) + twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")]) + twistd_args.extend(config.twistd_args) + twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin + + twistd_config = MyTwistdConfig() + try: + twistd_config.parseOptions(twistd_args) + except usage.error, ue: + # these arguments were unsuitable for 'twistd' + print >>err, config + print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue) + return 1 + twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} + + # On Unix-like platforms: + # Unless --nodaemon was provided, the twistd.runApp() below spawns off a + # child process, and the parent calls os._exit(0), so there's no way for + # us to get control afterwards, even with 'except SystemExit'. If + # application setup fails (e.g. ImportError), runApp() will raise an + # exception. + # + # So if we wanted to do anything with the running child, we'd have two + # options: + # + # * fork first, and have our child wait for the runApp() child to get + # running. (note: just fork(). This is easier than fork+exec, since we + # don't have to get PATH and PYTHONPATH set up, since we're not + # starting a *different* process, just cloning a new instance of the + # current process) + # * or have the user run a separate command some time after this one + # exits. + # + # For Tahoe, we don't need to do anything with the child, so we can just + # let it exit. + # + # On Windows: + # twistd does not fork; it just runs in the current process whether or not + # --nodaemon is specified. (As on Unix, --nodaemon does have the side effect + # of causing us to log to stdout/stderr.) + + if "--nodaemon" in twistd_args or sys.platform == "win32": + verb = "running" + else: + verb = "starting" + + print >>out, "%s node in %s" % (verb, quoted_basedir) + twistd.runApp(twistd_config) + # we should only reach here if --nodaemon or equivalent was used + return 0 diff --git a/src/allmydata/scripts/tahoe_restart.py b/src/allmydata/scripts/tahoe_restart.py new file mode 100644 index 000000000..f4fa087f2 --- /dev/null +++ b/src/allmydata/scripts/tahoe_restart.py @@ -0,0 +1,18 @@ +from .tahoe_start import StartOptions, start +from .tahoe_stop import stop, COULD_NOT_STOP + + +class RestartOptions(StartOptions): + subcommand_name = "restart" + + +def restart(config): + stderr = config.stderr + rc = stop(config) + if rc == COULD_NOT_STOP: + print >>stderr, "ignoring couldn't-stop" + rc = 0 + if rc: + print >>stderr, "not restarting" + return rc + return start(config) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py new file mode 100644 index 000000000..b5a0b9dfe --- /dev/null +++ b/src/allmydata/scripts/tahoe_run.py @@ -0,0 +1,10 @@ +from .tahoe_daemonize import daemonize, DaemonizeOptions + + +class RunOptions(DaemonizeOptions): + subcommand_name = "run" + + +def run(config): + config.twistd_args = config.twistd_args + ("--nodaemon",) + return daemonize(config) diff --git a/src/allmydata/scripts/tahoe_start.py b/src/allmydata/scripts/tahoe_start.py new file mode 100644 index 000000000..f1dd6f3cd --- /dev/null +++ b/src/allmydata/scripts/tahoe_start.py @@ -0,0 +1,139 @@ +import os +import io +import sys +import time +import subprocess +from os.path import join, exists + +from allmydata.scripts.common import BasedirOptions +from allmydata.scripts.default_nodedir import _default_nodedir +from allmydata.util.encodingutil import quote_local_unicode_path + +from .tahoe_daemonize import MyTwistdConfig, identify_node_type + + +class StartOptions(BasedirOptions): + subcommand_name = "start" + optParameters = [ + ("basedir", "C", None, + "Specify which Tahoe base directory should be used." + " This has the same effect as the global --node-directory option." + " [default: %s]" % quote_local_unicode_path(_default_nodedir)), + ] + + def parseArgs(self, basedir=None, *twistd_args): + # This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon' + # looks like an option to the tahoe subcommand, not to twistd. So you + # can either use 'tahoe start' or 'tahoe start NODEDIR + # --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start + # --TWISTD-OPTIONS' also isn't allowed, unfortunately. + + BasedirOptions.parseArgs(self, basedir) + self.twistd_args = twistd_args + + def getSynopsis(self): + return ("Usage: %s [global-options] %s [options]" + " [NODEDIR [twistd-options]]" + % (self.command_name, self.subcommand_name)) + + def getUsage(self, width=None): + t = BasedirOptions.getUsage(self, width) + "\n" + twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0] + t += twistd_options.replace("Options:", "twistd-options:", 1) + t += """ + +Note that if any twistd-options are used, NODEDIR must be specified explicitly +(not by default or using -C/--basedir or -d/--node-directory), and followed by +the twistd-options. +""" + return t + + +def start(config): + """ + Start a tahoe node (daemonize it and confirm startup) + + We run 'tahoe daemonize' with all the options given to 'tahoe + start' and then watch the log files for the correct text to appear + (e.g. "introducer started"). If that doesn't happen within a few + seconds, an error is printed along with all collected logs. + """ + + out = config.stdout + err = config.stderr + basedir = config['basedir'] + quoted_basedir = quote_local_unicode_path(basedir) + print >>out, "STARTING", quoted_basedir + if not os.path.isdir(basedir): + print >>err, "%s does not look like a directory at all" % quoted_basedir + return 1 + nodetype = identify_node_type(basedir) + if not nodetype: + print >>err, "%s is not a recognizable node directory" % quoted_basedir + return 1 + + # "tahoe start" attempts to monitor the logs for successful + # startup -- but we can't always do that. + + can_monitor_logs = False + if (nodetype in (u"client", u"introducer") + and "--nodaemon" not in config.twistd_args + and "--syslog" not in config.twistd_args + and "--logfile" not in config.twistd_args): + can_monitor_logs = True + + if "--help" in config.twistd_args: + return 0 + + if not can_monitor_logs: + print >>out, "Custom logging options; can't monitor logs for proper startup messages" + return 1 + + # before we spawn tahoe, we check if "the log file" exists or not, + # and if so remember how big it is -- essentially, we're doing + # "tail -f" to see what "this" incarnation of "tahoe daemonize" + # spews forth. + starting_offset = 0 + log_fname = join(basedir, 'logs', 'twistd.log') + if exists(log_fname): + with open(log_fname, 'r') as f: + f.seek(0, 2) + starting_offset = f.tell() + + # spawn tahoe. Note that since this daemonizes, it should return + # "pretty fast" and with a zero return-code, or else something + # Very Bad has happened. + try: + args = [sys.executable] + for i, arg in enumerate(sys.argv): + if arg in ['start', 'restart']: + args.append('daemonize') + else: + args.append(arg) + subprocess.check_call(args) + except subprocess.CalledProcessError as e: + return e.returncode + + # now, we have to determine if tahoe has actually started up + # successfully or not. so, we start sucking up log files and + # looking for "the magic string", which depends on the node type. + + magic_string = u'{} running'.format(nodetype) + with io.open(log_fname, 'r') as f: + f.seek(starting_offset) + + collected = u'' + start = time.time() + while time.time() - start < 5: + collected += f.read() + if magic_string in collected: + if not config.parent['quiet']: + print >>out, "Node has started successfully" + return 0 + time.sleep(0.1) + + print >>out, "Something has gone wrong starting the node." + print >>out, "Logs are available in '{}'".format(log_fname) + print >>out, "Collected for this run:" + print >>out, collected + return 1 diff --git a/src/allmydata/scripts/tahoe_stop.py b/src/allmydata/scripts/tahoe_stop.py new file mode 100644 index 000000000..6124dc672 --- /dev/null +++ b/src/allmydata/scripts/tahoe_stop.py @@ -0,0 +1,88 @@ +import os +import time +import signal + +from allmydata.scripts.common import BasedirOptions +from allmydata.util.encodingutil import quote_local_unicode_path + +COULD_NOT_STOP = 2 + + +class StopOptions(BasedirOptions): + def parseArgs(self, basedir=None): + BasedirOptions.parseArgs(self, basedir) + + def getSynopsis(self): + return ("Usage: %s [global-options] stop [options] [NODEDIR]" + % (self.command_name,)) + + +def stop(config): + out = config.stdout + err = config.stderr + basedir = config['basedir'] + quoted_basedir = quote_local_unicode_path(basedir) + print >>out, "STOPPING", quoted_basedir + pidfile = os.path.join(basedir, u"twistd.pid") + if not os.path.exists(pidfile): + print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir + # we define rc=2 to mean "nothing is running, but it wasn't me who + # stopped it" + return COULD_NOT_STOP + with open(pidfile, "r") as f: + pid = f.read() + + try: + pid = int(pid) + except ValueError: + # The error message below mimics a Twisted error message, which is + # displayed when starting a node with an invalid pidfile. + print >>err, "Pidfile %s contains non-numeric value" % pidfile + # we define rc=2 to mean "nothing is running, but it wasn't me who + # stopped it" + return 2 + + # kill it hard (SIGKILL), delete the twistd.pid file, then wait for the + # process itself to go away. If it hasn't gone away after 20 seconds, warn + # the user but keep waiting until they give up. + try: + os.kill(pid, signal.SIGKILL) + except OSError, oserr: + if oserr.errno == 3: + print oserr.strerror + # the process didn't exist, so wipe the pid file + os.remove(pidfile) + return COULD_NOT_STOP + else: + raise + try: + os.remove(pidfile) + except EnvironmentError: + pass + start = time.time() + time.sleep(0.1) + wait = 40 + first_time = True + while True: + # poll once per second until we see the process is no longer running + try: + os.kill(pid, 0) + except OSError: + print >>out, "process %d is dead" % pid + return + wait -= 1 + if wait < 0: + if first_time: + print >>err, ("It looks like pid %d is still running " + "after %d seconds" % (pid, + (time.time() - start))) + print >>err, "I will keep watching it until you interrupt me." + wait = 10 + first_time = False + else: + print >>err, "pid %d still running after %d seconds" % \ + (pid, (time.time() - start)) + wait = 10 + time.sleep(1) + # we define rc=1 to mean "I think something is still running, sorry" + return 1 diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 1f39e8836..31c6f5f05 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -20,12 +20,14 @@ import allmydata.scripts.common_http from pycryptopp.publickey import ed25519 # Test that the scripts can be imported. -from allmydata.scripts import create_node, debug, startstop_node, \ +from allmydata.scripts import create_node, debug, tahoe_start, tahoe_restart, \ tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls, \ - tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen -_hush_pyflakes = [create_node, debug, startstop_node, + tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen, \ + tahoe_stop, tahoe_daemonize, tahoe_run +_hush_pyflakes = [create_node, debug, tahoe_start, tahoe_restart, tahoe_stop, tahoe_add_alias, tahoe_backup, tahoe_check, tahoe_cp, tahoe_get, tahoe_ls, - tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen] + tahoe_manifest, tahoe_mkdir, tahoe_mv, tahoe_put, tahoe_unlink, tahoe_webopen, + tahoe_daemonize, tahoe_run] from allmydata.scripts import common from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \ @@ -630,19 +632,19 @@ class Help(unittest.TestCase): self.failUnlessIn("[options]", help) def test_start(self): - help = str(startstop_node.StartOptions()) + help = str(tahoe_start.StartOptions()) self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help) def test_stop(self): - help = str(startstop_node.StopOptions()) + help = str(tahoe_stop.StopOptions()) self.failUnlessIn("[options] [NODEDIR]", help) def test_restart(self): - help = str(startstop_node.RestartOptions()) + help = str(tahoe_restart.RestartOptions()) self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help) def test_run(self): - help = str(startstop_node.RunOptions()) + help = str(tahoe_run.RunOptions()) self.failUnlessIn("[options] [NODEDIR [twistd-options]]", help) def test_create_client(self): @@ -1303,11 +1305,11 @@ class Stop(unittest.TestCase): basedir.makedirs() basedir.child(u"twistd.pid").setContent(b"foo") - config = startstop_node.StopOptions() + config = tahoe_stop.StopOptions() config.stdout = StringIO() config.stderr = StringIO() config['basedir'] = basedir.path - result_code = startstop_node.stop(config) + result_code = tahoe_stop.stop(config) self.assertEqual(2, result_code) self.assertIn("contains non-numeric value", config.stderr.getvalue()) diff --git a/src/allmydata/test/cli/test_daemonize.py b/src/allmydata/test/cli/test_daemonize.py new file mode 100644 index 000000000..3853054e4 --- /dev/null +++ b/src/allmydata/test/cli/test_daemonize.py @@ -0,0 +1,177 @@ +import os +from os.path import dirname, join +from mock import patch, Mock +from StringIO import StringIO +from sys import getfilesystemencoding +from twisted.trial import unittest +from allmydata.scripts import runner +from allmydata.scripts.tahoe_daemonize import identify_node_type +from allmydata.scripts.tahoe_daemonize import DaemonizeTahoeNodePlugin +from allmydata.scripts.tahoe_daemonize import DaemonizeOptions + + +class Util(unittest.TestCase): + + def test_node_type_nothing(self): + tmpdir = self.mktemp() + base = dirname(tmpdir).decode(getfilesystemencoding()) + + t = identify_node_type(base) + + self.assertIs(None, t) + + def test_node_type_introducer(self): + tmpdir = self.mktemp() + base = dirname(tmpdir).decode(getfilesystemencoding()) + with open(join(dirname(tmpdir), 'introducer.tac'), 'w') as f: + f.write("test placeholder") + + t = identify_node_type(base) + + self.assertEqual(u"introducer", t) + + def test_daemonize(self): + tmpdir = self.mktemp() + plug = DaemonizeTahoeNodePlugin('client', tmpdir) + + with patch('twisted.internet.reactor') as r: + def call(fn, *args, **kw): + fn() + r.callWhenRunning = call + service = plug.makeService(None) + service.parent = Mock() + service.startService() + + self.assertTrue(service is not None) + + def test_daemonize_no_keygen(self): + tmpdir = self.mktemp() + plug = DaemonizeTahoeNodePlugin('key-generator', tmpdir) + + with patch('twisted.internet.reactor') as r: + def call(fn, *args, **kw): + fn() + r.callWhenRunning = call + service = plug.makeService(None) + service.parent = Mock() + with self.assertRaises(ValueError) as ctx: + service.startService() + self.assertIn( + "key-generator support removed", + str(ctx.exception) + ) + + def test_daemonize_unknown_nodetype(self): + tmpdir = self.mktemp() + plug = DaemonizeTahoeNodePlugin('an-unknown-service', tmpdir) + + with patch('twisted.internet.reactor') as r: + def call(fn, *args, **kw): + fn() + r.callWhenRunning = call + service = plug.makeService(None) + service.parent = Mock() + with self.assertRaises(ValueError) as ctx: + service.startService() + self.assertIn( + "unknown nodetype", + str(ctx.exception) + ) + + def test_daemonize_options(self): + parent = runner.Options() + opts = DaemonizeOptions() + opts.parent = parent + opts.parseArgs() + + # just gratuitous coverage, ensureing we don't blow up on + # these methods. + opts.getSynopsis() + opts.getUsage() + + +class RunDaemonizeTests(unittest.TestCase): + + def setUp(self): + # no test should change our working directory + self._working = os.path.abspath('.') + d = super(RunDaemonizeTests, self).setUp() + self._reactor = patch('twisted.internet.reactor') + self._twistd = patch('allmydata.scripts.tahoe_daemonize.twistd') + self.node_dir = self.mktemp() + os.mkdir(self.node_dir) + for cm in [self._reactor, self._twistd]: + cm.__enter__() + return d + + def tearDown(self): + d = super(RunDaemonizeTests, self).tearDown() + for cm in [self._reactor, self._twistd]: + cm.__exit__(None, None, None) + # Note: if you raise an exception (e.g. via self.assertEqual + # or raise RuntimeError) it is apparently just ignored and the + # test passes anyway... + if self._working != os.path.abspath('.'): + print("WARNING: a test just changed the working dir; putting it back") + os.chdir(self._working) + return d + + def _placeholder_nodetype(self, nodetype): + fname = join(self.node_dir, '{}.tac'.format(nodetype)) + with open(fname, 'w') as f: + f.write("test placeholder") + + def test_daemonize_defaults(self): + self._placeholder_nodetype('introducer') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't much around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'daemonize', + ]) + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + runner.dispatch(config, i, o, e) + + self.assertEqual(0, exit_code[0]) + + def test_daemonize_wrong_nodetype(self): + self._placeholder_nodetype('invalid') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't much around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'daemonize', + ]) + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + runner.dispatch(config, i, o, e) + + self.assertEqual(0, exit_code[0]) + + def test_daemonize_run(self): + self._placeholder_nodetype('client') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't much around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'daemonize', + ]) + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + from allmydata.scripts.tahoe_daemonize import daemonize + daemonize(config) diff --git a/src/allmydata/test/cli/test_start.py b/src/allmydata/test/cli/test_start.py new file mode 100644 index 000000000..34ca15032 --- /dev/null +++ b/src/allmydata/test/cli/test_start.py @@ -0,0 +1,209 @@ +import os +import shutil +import subprocess +from os.path import join +from mock import patch +from StringIO import StringIO + +from twisted.trial import unittest +from allmydata.scripts import runner + + +#@patch('twisted.internet.reactor') +@patch('allmydata.scripts.tahoe_start.subprocess') +class RunStartTests(unittest.TestCase): + + def setUp(self): + d = super(RunStartTests, self).setUp() + self.node_dir = self.mktemp() + os.mkdir(self.node_dir) + return d + + def _placeholder_nodetype(self, nodetype): + fname = join(self.node_dir, '{}.tac'.format(nodetype)) + with open(fname, 'w') as f: + f.write("test placeholder") + + def _pid_file(self, pid): + fname = join(self.node_dir, 'twistd.pid') + with open(fname, 'w') as f: + f.write(u"{}\n".format(pid)) + + def _logs(self, logs): + os.mkdir(join(self.node_dir, 'logs')) + fname = join(self.node_dir, 'logs', 'twistd.log') + with open(fname, 'w') as f: + f.write(logs) + + def test_start_defaults(self, _subprocess): + self._placeholder_nodetype('client') + self._pid_file(1234) + self._logs('one log\ntwo log\nred log\nblue log\n') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + ]) + i, o, e = StringIO(), StringIO(), StringIO() + try: + with patch('allmydata.scripts.tahoe_start.os'): + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + + def launch(*args, **kw): + with open(join(self.node_dir, 'logs', 'twistd.log'), 'a') as f: + f.write('client running\n') # "the magic" + _subprocess.check_call = launch + runner.dispatch(config, i, o, e) + except Exception: + pass + + self.assertEqual([0], exit_code) + self.assertTrue('Node has started' in o.getvalue()) + + def test_start_fails(self, _subprocess): + self._placeholder_nodetype('client') + self._logs('existing log line\n') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + ]) + + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.tahoe_start.time') as t: + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + + thetime = [0] + def _time(): + thetime[0] += 0.1 + return thetime[0] + t.time = _time + + def launch(*args, **kw): + with open(join(self.node_dir, 'logs', 'twistd.log'), 'a') as f: + f.write('a new log line\n') + _subprocess.check_call = launch + + runner.dispatch(config, i, o, e) + + # should print out the collected logs and an error-code + self.assertTrue("a new log line" in o.getvalue()) + self.assertEqual([1], exit_code) + + def test_start_subprocess_fails(self, _subprocess): + self._placeholder_nodetype('client') + self._logs('existing log line\n') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + ]) + + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.tahoe_start.time'): + with patch('allmydata.scripts.runner.sys') as s: + # undo patch for the exception-class + _subprocess.CalledProcessError = subprocess.CalledProcessError + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + + def launch(*args, **kw): + raise subprocess.CalledProcessError(42, "tahoe") + _subprocess.check_call = launch + + runner.dispatch(config, i, o, e) + + # should get our "odd" error-code + self.assertEqual([42], exit_code) + + def test_start_help(self, _subprocess): + self._placeholder_nodetype('client') + + std = StringIO() + with patch('sys.stdout') as stdo: + stdo.write = std.write + try: + runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + '--help', + ], stdout=std) + self.fail("Should get exit") + except SystemExit as e: + print(e) + + self.assertIn( + "Usage:", + std.getvalue() + ) + + def test_start_unknown_node_type(self, _subprocess): + self._placeholder_nodetype('bogus') + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + ]) + + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + + runner.dispatch(config, i, o, e) + + # should print out the collected logs and an error-code + self.assertIn( + "is not a recognizable node directory", + e.getvalue() + ) + self.assertEqual([1], exit_code) + + def test_start_nodedir_not_dir(self, _subprocess): + shutil.rmtree(self.node_dir) + assert not os.path.isdir(self.node_dir) + + config = runner.parse_or_exit_with_explanation([ + # have to do this so the tests don't muck around in + # ~/.tahoe (the default) + '--node-directory', self.node_dir, + 'start', + ]) + + i, o, e = StringIO(), StringIO(), StringIO() + with patch('allmydata.scripts.runner.sys') as s: + exit_code = [None] + def _exit(code): + exit_code[0] = code + s.exit = _exit + + runner.dispatch(config, i, o, e) + + # should print out the collected logs and an error-code + self.assertIn( + "does not look like a directory at all", + e.getvalue() + ) + self.assertEqual([1], exit_code) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index b118357bc..74ed51c28 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -860,6 +860,20 @@ class FakeServerTracker: class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, ShouldFailMixin): + + def setUp(self): + d = super(EncodingParameters, self).setUp() + self._curdir = os.path.abspath(os.path.curdir) + return d + + def tearDown(self): + d = super(EncodingParameters, self).tearDown() + self.assertEqual( + os.path.abspath(os.path.curdir), + self._curdir, + ) + return d + def find_all_shares(self, unused=None): """Locate shares on disk. Returns a dict that maps server to set of sharenums.