tests: use shared run_cli()/do_cli()

A couple of test classes which defined their own flavors were changed to
use the common one.

src/allmydata/test/cli/test_cli.py

@@ -29,10 +29,10 @@ from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \
      DefaultAliasMarker
 
 from allmydata.scripts import cli, debug, runner
-from ..common_util import ReallyEqualMixin, skip_if_cannot_represent_filename
+from ..common_util import (ReallyEqualMixin, skip_if_cannot_represent_filename,
+                           run_cli)
 from ..no_network import GridTestMixin
 from .common import CLITestMixin, parse_options
-from twisted.internet import threads # CLI tests use deferToThread
 from twisted.python import usage
 
 from allmydata.util.encodingutil import listdir_unicode, get_io_encoding
@@ -705,21 +705,9 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase):
 
 
 class Admin(unittest.TestCase):
-    def do_cli(self, *args, **kwargs):
-        argv = list(args)
-        stdin = kwargs.get("stdin", "")
-        stdout, stderr = StringIO(), StringIO()
-        d = threads.deferToThread(runner.runner, argv, run_by_human=False,
-                                  stdin=StringIO(stdin),
-                                  stdout=stdout, stderr=stderr)
-        def _done(res):
-            return stdout.getvalue(), stderr.getvalue()
-        d.addCallback(_done)
-        return d
-
     def test_generate_keypair(self):
-        d = self.do_cli("admin", "generate-keypair")
+        d = run_cli("admin", "generate-keypair")
-        def _done( (stdout, stderr) ):
+        def _done( (rc, stdout, stderr) ):
             lines = [line.strip() for line in stdout.splitlines()]
             privkey_bits = lines[0].split()
             pubkey_bits = lines[1].split()
@@ -738,8 +726,8 @@ class Admin(unittest.TestCase):
 
     def test_derive_pubkey(self):
         priv1,pub1 = keyutil.make_keypair()
-        d = self.do_cli("admin", "derive-pubkey", priv1)
+        d = run_cli("admin", "derive-pubkey", priv1)
-        def _done( (stdout, stderr) ):
+        def _done( (rc, stdout, stderr) ):
             lines = stdout.split("\n")
             privkey_line = lines[0].strip()
             pubkey_line = lines[1].strip()

src/allmydata/test/cli/test_create.py

@@ -1,54 +1,53 @@
 import os
-from StringIO import StringIO
 from twisted.trial import unittest
-from allmydata.scripts import runner
+from twisted.internet import defer
 from allmydata.util import configutil
+from ..common_util import run_cli
 
 class Config(unittest.TestCase):
-    def do_cli(self, *args):
-        argv = list(args)
-        stdout, stderr = StringIO(), StringIO()
-        rc = runner.runner(argv, run_by_human=False,
-                           stdout=stdout, stderr=stderr)
-        return rc, stdout.getvalue(), stderr.getvalue()
-
     def read_config(self, basedir):
         tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
         config = configutil.get_config(tahoe_cfg)
         return config
 
+    @defer.inlineCallbacks
     def test_client(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-client", basedir)
+        rc, out, err = yield run_cli("create-client", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
 
+    @defer.inlineCallbacks
     def test_client_hide_ip(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-client", "--hide-ip", basedir)
+        rc, out, err = yield run_cli("create-client", "--hide-ip", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)
 
+    @defer.inlineCallbacks
     def test_node(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-node", basedir)
+        rc, out, err = yield run_cli("create-node", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
 
+    @defer.inlineCallbacks
     def test_node_hide_ip(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-node", "--hide-ip", basedir)
+        rc, out, err = yield run_cli("create-node", "--hide-ip", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)
 
+    @defer.inlineCallbacks
     def test_introducer(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-introducer", basedir)
+        rc, out, err = yield run_cli("create-introducer", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
 
+    @defer.inlineCallbacks
     def test_introducer_hide_ip(self):
         basedir = self.mktemp()
-        rc, out, err = self.do_cli("create-introducer", "--hide-ip", basedir)
+        rc, out, err = yield run_cli("create-introducer", "--hide-ip", basedir)
         cfg = self.read_config(basedir)
         self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)

src/allmydata/test/test_deepcheck.py

@@ -1,15 +1,12 @@
 
 import os, simplejson, urllib
-from cStringIO import StringIO
 from twisted.trial import unittest
 from twisted.internet import defer
-from twisted.internet import threads # CLI tests use deferToThread
 from allmydata.immutable import upload
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.mutable.publish import MutableData
 from allmydata.util import idlib
 from allmydata.util import base32
-from allmydata.scripts import runner
 from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
      IDeepCheckResults, IDeepCheckAndRepairResults
 from allmydata.monitor import Monitor, OperationCancelledError
@@ -18,19 +15,13 @@ from twisted.web.client import getPage
 
 from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
      ShouldFailMixin
-from allmydata.test.common_util import StallMixin
+from .common_util import StallMixin, run_cli
 from allmydata.test.no_network import GridTestMixin
+from .cli.common import CLITestMixin
 
 timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
 
 class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
-    def _run_cli(self, argv):
-        stdout, stderr = StringIO(), StringIO()
-        # this can only do synchronous operations
-        assert argv[0] == "debug"
-        runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
-        return stdout.getvalue()
-
     def test_good(self):
         self.basedir = "deepcheck/MutableChecker/good"
         self.set_up_grid()
@@ -130,7 +121,8 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
         return d
 
 
-class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
+class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
+                    CLITestMixin):
 
     def web_json(self, n, **kwargs):
         kwargs["output"] = "json"
@@ -727,17 +719,6 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
 
         return d
 
-    def _run_cli(self, argv, stdin=""):
-        #print "CLI:", argv
-        stdout, stderr = StringIO(), StringIO()
-        d = threads.deferToThread(runner.runner, argv, run_by_human=False,
-                                  stdin=StringIO(stdin),
-                                  stdout=stdout, stderr=stderr)
-        def _done(res):
-            return stdout.getvalue(), stderr.getvalue()
-        d.addCallback(_done)
-        return d
-
     def do_test_cli_good(self, ignored):
         d = defer.succeed(None)
         d.addCallback(lambda ign: self.do_cli_manifest_stream1())
@@ -757,11 +738,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
 
     def do_cli_manifest_stream1(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("manifest", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check((rc,out,err)):
-                           "manifest",
-                           self.root_uri])
-        def _check((out,err)):
             self.failUnlessEqual(err, "")
             lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 8)
@@ -785,12 +763,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def do_cli_manifest_stream2(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("manifest", "--raw", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check((rc,out,err)):
-                           "manifest",
-                           "--raw",
-                           self.root_uri])
-        def _check((out,err)):
             self.failUnlessEqual(err, "")
             # this should be the same as the POST t=stream-manifest output
             self._check_streamed_manifest(out)
@@ -798,24 +772,16 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def do_cli_manifest_stream3(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("manifest", "--storage-index", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check((rc,out,err)):
-                           "manifest",
-                           "--storage-index",
-                           self.root_uri])
-        def _check((out,err)):
             self.failUnlessEqual(err, "")
             self._check_manifest_storage_index(out)
         d.addCallback(_check)
         return d
 
     def do_cli_manifest_stream4(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("manifest", "--verify-cap", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check((rc,out,err)):
-                           "manifest",
-                           "--verify-cap",
-                           self.root_uri])
-        def _check((out,err)):
             self.failUnlessEqual(err, "")
             lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 3)
@@ -826,12 +792,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def do_cli_manifest_stream5(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("manifest", "--repair-cap", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check((rc,out,err)):
-                           "manifest",
-                           "--repair-cap",
-                           self.root_uri])
-        def _check((out,err)):
             self.failUnlessEqual(err, "")
             lines = [l for l in out.split("\n") if l]
             self.failUnlessEqual(len(lines), 3)
@@ -842,11 +804,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def do_cli_stats1(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("stats", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check3((rc,out,err)):
-                           "stats",
-                           self.root_uri])
-        def _check3((out,err)):
             lines = [l.strip() for l in out.split("\n") if l]
             self.failUnless("count-immutable-files: 1" in lines)
             self.failUnless("count-mutable-files: 1" in lines)
@@ -862,12 +821,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
         return d
 
     def do_cli_stats2(self):
-        basedir = self.get_clientdir(0)
+        d = self.do_cli("stats", "--raw", self.root_uri)
-        d = self._run_cli(["--node-directory", basedir,
+        def _check4((rc,out,err)):
-                           "stats",
-                           "--raw",
-                           self.root_uri])
-        def _check4((out,err)):
             data = simplejson.loads(out)
             self.failUnlessEqual(data["count-immutable-files"], 1)
             self.failUnlessEqual(data["count-immutable-files"], 1)
@@ -983,20 +938,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
 
         return d
 
-    def _run_cli(self, argv):
-        stdout, stderr = StringIO(), StringIO()
-        # this can only do synchronous operations
-        assert argv[0] == "debug"
-        runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
-        return stdout.getvalue()
-
     def _delete_some_shares(self, node):
         self.delete_shares_numbered(node.get_uri(), [0,1])
 
+    @defer.inlineCallbacks
     def _corrupt_some_shares(self, node):
         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
             if shnum in (0,1):
-                self._run_cli(["debug", "corrupt-share", sharefile])
+                yield run_cli("debug", "corrupt-share", sharefile)
 
     def _delete_most_shares(self, node):
         self.delete_shares_numbered(node.get_uri(), range(1,10))

src/allmydata/test/test_runner.py

@@ -1,5 +1,4 @@
 import os.path, re, sys, subprocess
-from cStringIO import StringIO
 
 from twisted.trial import unittest
 
@@ -15,6 +14,7 @@ from allmydata.client import Client
 from allmydata.test import common_util
 import allmydata
 from allmydata import __appname__
+from .common_util import run_cli
 
 
 timeout = 240
@@ -180,11 +180,7 @@ class CreateNode(unittest.TestCase):
         fileutil.make_dirs(basedir)
         return basedir
 
-    def run_tahoe(self, argv):
+    @inlineCallbacks
-        out,err = StringIO(), StringIO()
-        rc = runner.runner(argv, stdout=out, stderr=err)
-        return rc, out.getvalue(), err.getvalue()
-
     def do_create(self, kind, *args):
         basedir = self.workdir("test_" + kind)
         command = "create-" + kind
@@ -193,7 +189,7 @@ class CreateNode(unittest.TestCase):
 
         n1 = os.path.join(basedir, command + "-n1")
         argv = ["--quiet", command, "--basedir", n1] + list(args)
-        rc, out, err = self.run_tahoe(argv)
+        rc, out, err = yield run_cli(*argv)
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -213,7 +209,7 @@ class CreateNode(unittest.TestCase):
                 self.failUnless("\nreserved_space = 1G\n" in content)
 
         # creating the node a second time should be rejected
-        rc, out, err = self.run_tahoe(argv)
+        rc, out, err = yield run_cli(*argv)
         self.failIfEqual(rc, 0, str((out, err, rc)))
         self.failUnlessEqual(out, "")
         self.failUnless("is not empty." in err)
@@ -226,7 +222,7 @@ class CreateNode(unittest.TestCase):
         # test that the non --basedir form works too
         n2 = os.path.join(basedir, command + "-n2")
         argv = ["--quiet", command] + list(args) + [n2]
-        rc, out, err = self.run_tahoe(argv)
+        rc, out, err = yield run_cli(*argv)
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -236,7 +232,7 @@ class CreateNode(unittest.TestCase):
         # test the --node-directory form
         n3 = os.path.join(basedir, command + "-n3")
         argv = ["--quiet", "--node-directory", n3, command] + list(args)
-        rc, out, err = self.run_tahoe(argv)
+        rc, out, err = yield run_cli(*argv)
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -247,7 +243,7 @@ class CreateNode(unittest.TestCase):
             # test that the output (without --quiet) includes the base directory
             n4 = os.path.join(basedir, command + "-n4")
             argv = [command] + list(args) + [n4]
-            rc, out, err = self.run_tahoe(argv)
+            rc, out, err = yield run_cli(*argv)
             self.failUnlessEqual(err, "")
             self.failUnlessIn(" created in ", out)
             self.failUnlessIn(n4, out)

src/allmydata/test/test_system.py

@@ -1,10 +1,8 @@
 
 import os, re, sys, time, simplejson
-from cStringIO import StringIO
 
 from twisted.trial import unittest
 from twisted.internet import defer
-from twisted.internet import threads # CLI tests use deferToThread
 from twisted.application import service
 
 import allmydata
@@ -20,7 +18,6 @@ from allmydata.util import log, base32
 from allmydata.util.encodingutil import quote_output, unicode_to_argv
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.consumer import MemoryConsumer, download_to_data
-from allmydata.scripts import runner
 from allmydata.stats import StatsGathererService
 from allmydata.interfaces import IDirectoryNode, IFileNode, \
      NoSuchChildError, NoSharesError
@@ -39,6 +36,7 @@ from .common import TEST_RSA_KEY_SIZE
 # TODO: move this to common or common_util
 from allmydata.test.test_runner import RunBinTahoeMixin
 from . import common_util as testutil
+from .common_util import run_cli
 
 LARGE_DATA = """
 This is some data to publish to the remote grid.., which needs to be large
@@ -1064,6 +1062,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             return d1
         d.addCallback(_create_mutable)
 
+        @defer.inlineCallbacks
         def _test_debug(res):
             # find a share. It is important to run this while there is only
             # one slot in the grid.
@@ -1073,11 +1072,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                     % filename)
             log.msg(" for clients[%d]" % client_num)
 
-            out,err = StringIO(), StringIO()
+            rc,output,err = yield run_cli("debug", "dump-share", "--offsets",
-            rc = runner.runner(["debug", "dump-share", "--offsets",
+                                          filename)
-                                filename],
-                               stdout=out, stderr=err)
-            output = out.getvalue()
             self.failUnlessEqual(rc, 0)
             try:
                 self.failUnless("Mutable slot found:\n" in output)
@@ -1862,6 +1858,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         return d
 
+    @defer.inlineCallbacks
     def _test_runner(self, res):
         # exercise some of the diagnostic tools in runner.py
 
@@ -1887,11 +1884,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
                       % self.basedir)
         log.msg("test_system.SystemTest._test_runner using %r" % filename)
 
-        out,err = StringIO(), StringIO()
+        rc,output,err = yield run_cli("debug", "dump-share", "--offsets",
-        rc = runner.runner(["debug", "dump-share", "--offsets",
+                                      unicode_to_argv(filename))
-                            unicode_to_argv(filename)],
-                           stdout=out, stderr=err)
-        output = out.getvalue()
         self.failUnlessEqual(rc, 0)
 
         # we only upload a single file, so we can assert some things about
@@ -1917,23 +1911,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         sharedir, shnum = os.path.split(filename)
         storagedir, storage_index_s = os.path.split(sharedir)
         storage_index_s = str(storage_index_s)
-        out,err = StringIO(), StringIO()
         nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
-        cmd = ["debug", "find-shares", storage_index_s] + nodedirs
+        rc,out,err = yield run_cli("debug", "find-shares", storage_index_s,
-        rc = runner.runner(cmd, stdout=out, stderr=err)
+                                   *nodedirs)
         self.failUnlessEqual(rc, 0)
-        out.seek(0)
+        sharefiles = [sfn.strip() for sfn in out.splitlines()]
-        sharefiles = [sfn.strip() for sfn in out.readlines()]
         self.failUnlessEqual(len(sharefiles), 10)
 
         # also exercise the 'catalog-shares' tool
-        out,err = StringIO(), StringIO()
         nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
-        cmd = ["debug", "catalog-shares"] + nodedirs
+        rc,out,err = yield run_cli("debug", "catalog-shares", *nodedirs)
-        rc = runner.runner(cmd, stdout=out, stderr=err)
         self.failUnlessEqual(rc, 0)
-        out.seek(0)
+        descriptions = [sfn.strip() for sfn in out.splitlines()]
-        descriptions = [sfn.strip() for sfn in out.readlines()]
         self.failUnlessEqual(len(descriptions), 30)
         matching = [line
                     for line in descriptions
@@ -1981,10 +1970,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         f.write(private_uri)
         f.close()
 
+        @defer.inlineCallbacks
         def run(ignored, verb, *args, **kwargs):
-            stdin = kwargs.get("stdin", "")
+            rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
-            newargs = nodeargs + [verb] + list(args)
+            defer.returnValue((out,err))
-            return self._run_cli(newargs, stdin=stdin)
 
         def _check_ls((out,err), expected_children, unexpected_children=[]):
             self.failUnlessEqual(err, "")
@@ -2345,17 +2334,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(_check_ls)
         return d
 
-    def _run_cli(self, argv, stdin=""):
-        #print "CLI:", argv
-        stdout, stderr = StringIO(), StringIO()
-        d = threads.deferToThread(runner.runner, argv, run_by_human=False,
-                                  stdin=StringIO(stdin),
-                                  stdout=stdout, stderr=stderr)
-        def _done(res):
-            return stdout.getvalue(), stderr.getvalue()
-        d.addCallback(_done)
-        return d
-
     def _test_checker(self, res):
         ut = upload.Data("too big to be literal" * 200, convergence=None)
         d = self._personal_node.add_file(u"big file", ut)