tests: use shared run_cli()/do_cli()

A couple of test classes which defined their own flavors were changed to
use the common one.
This commit is contained in:
Brian Warner 2016-09-09 10:41:20 -07:00
parent 07e4c491f5
commit 7193bff48b
5 changed files with 62 additions and 152 deletions

View File

@ -29,10 +29,10 @@ from allmydata.scripts.common import DEFAULT_ALIAS, get_aliases, get_alias, \
DefaultAliasMarker
from allmydata.scripts import cli, debug, runner
from ..common_util import ReallyEqualMixin, skip_if_cannot_represent_filename
from ..common_util import (ReallyEqualMixin, skip_if_cannot_represent_filename,
run_cli)
from ..no_network import GridTestMixin
from .common import CLITestMixin, parse_options
from twisted.internet import threads # CLI tests use deferToThread
from twisted.python import usage
from allmydata.util.encodingutil import listdir_unicode, get_io_encoding
@ -705,21 +705,9 @@ class Ln(GridTestMixin, CLITestMixin, unittest.TestCase):
class Admin(unittest.TestCase):
def do_cli(self, *args, **kwargs):
argv = list(args)
stdin = kwargs.get("stdin", "")
stdout, stderr = StringIO(), StringIO()
d = threads.deferToThread(runner.runner, argv, run_by_human=False,
stdin=StringIO(stdin),
stdout=stdout, stderr=stderr)
def _done(res):
return stdout.getvalue(), stderr.getvalue()
d.addCallback(_done)
return d
def test_generate_keypair(self):
d = self.do_cli("admin", "generate-keypair")
def _done( (stdout, stderr) ):
d = run_cli("admin", "generate-keypair")
def _done( (rc, stdout, stderr) ):
lines = [line.strip() for line in stdout.splitlines()]
privkey_bits = lines[0].split()
pubkey_bits = lines[1].split()
@ -738,8 +726,8 @@ class Admin(unittest.TestCase):
def test_derive_pubkey(self):
priv1,pub1 = keyutil.make_keypair()
d = self.do_cli("admin", "derive-pubkey", priv1)
def _done( (stdout, stderr) ):
d = run_cli("admin", "derive-pubkey", priv1)
def _done( (rc, stdout, stderr) ):
lines = stdout.split("\n")
privkey_line = lines[0].strip()
pubkey_line = lines[1].strip()

View File

@ -1,54 +1,53 @@
import os
from StringIO import StringIO
from twisted.trial import unittest
from allmydata.scripts import runner
from twisted.internet import defer
from allmydata.util import configutil
from ..common_util import run_cli
class Config(unittest.TestCase):
def do_cli(self, *args):
argv = list(args)
stdout, stderr = StringIO(), StringIO()
rc = runner.runner(argv, run_by_human=False,
stdout=stdout, stderr=stderr)
return rc, stdout.getvalue(), stderr.getvalue()
def read_config(self, basedir):
tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
config = configutil.get_config(tahoe_cfg)
return config
@defer.inlineCallbacks
def test_client(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-client", basedir)
rc, out, err = yield run_cli("create-client", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
@defer.inlineCallbacks
def test_client_hide_ip(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-client", "--hide-ip", basedir)
rc, out, err = yield run_cli("create-client", "--hide-ip", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)
@defer.inlineCallbacks
def test_node(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-node", basedir)
rc, out, err = yield run_cli("create-node", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
@defer.inlineCallbacks
def test_node_hide_ip(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-node", "--hide-ip", basedir)
rc, out, err = yield run_cli("create-node", "--hide-ip", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)
@defer.inlineCallbacks
def test_introducer(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-introducer", basedir)
rc, out, err = yield run_cli("create-introducer", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), True)
@defer.inlineCallbacks
def test_introducer_hide_ip(self):
basedir = self.mktemp()
rc, out, err = self.do_cli("create-introducer", "--hide-ip", basedir)
rc, out, err = yield run_cli("create-introducer", "--hide-ip", basedir)
cfg = self.read_config(basedir)
self.assertEqual(cfg.getboolean("node", "reveal-IP-address"), False)

View File

@ -1,15 +1,12 @@
import os, simplejson, urllib
from cStringIO import StringIO
from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet import threads # CLI tests use deferToThread
from allmydata.immutable import upload
from allmydata.mutable.common import UnrecoverableFileError
from allmydata.mutable.publish import MutableData
from allmydata.util import idlib
from allmydata.util import base32
from allmydata.scripts import runner
from allmydata.interfaces import ICheckResults, ICheckAndRepairResults, \
IDeepCheckResults, IDeepCheckAndRepairResults
from allmydata.monitor import Monitor, OperationCancelledError
@ -18,19 +15,13 @@ from twisted.web.client import getPage
from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
ShouldFailMixin
from allmydata.test.common_util import StallMixin
from .common_util import StallMixin, run_cli
from allmydata.test.no_network import GridTestMixin
from .cli.common import CLITestMixin
timeout = 2400 # One of these took 1046.091s on Zandr's ARM box.
class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def _run_cli(self, argv):
stdout, stderr = StringIO(), StringIO()
# this can only do synchronous operations
assert argv[0] == "debug"
runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
return stdout.getvalue()
def test_good(self):
self.basedir = "deepcheck/MutableChecker/good"
self.set_up_grid()
@ -130,7 +121,8 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
return d
class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin):
class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
CLITestMixin):
def web_json(self, n, **kwargs):
kwargs["output"] = "json"
@ -727,17 +719,6 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def _run_cli(self, argv, stdin=""):
#print "CLI:", argv
stdout, stderr = StringIO(), StringIO()
d = threads.deferToThread(runner.runner, argv, run_by_human=False,
stdin=StringIO(stdin),
stdout=stdout, stderr=stderr)
def _done(res):
return stdout.getvalue(), stderr.getvalue()
d.addCallback(_done)
return d
def do_test_cli_good(self, ignored):
d = defer.succeed(None)
d.addCallback(lambda ign: self.do_cli_manifest_stream1())
@ -757,11 +738,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
def do_cli_manifest_stream1(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"manifest",
self.root_uri])
def _check((out,err)):
d = self.do_cli("manifest", self.root_uri)
def _check((rc,out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 8)
@ -785,12 +763,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def do_cli_manifest_stream2(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"manifest",
"--raw",
self.root_uri])
def _check((out,err)):
d = self.do_cli("manifest", "--raw", self.root_uri)
def _check((rc,out,err)):
self.failUnlessEqual(err, "")
# this should be the same as the POST t=stream-manifest output
self._check_streamed_manifest(out)
@ -798,24 +772,16 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def do_cli_manifest_stream3(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"manifest",
"--storage-index",
self.root_uri])
def _check((out,err)):
d = self.do_cli("manifest", "--storage-index", self.root_uri)
def _check((rc,out,err)):
self.failUnlessEqual(err, "")
self._check_manifest_storage_index(out)
d.addCallback(_check)
return d
def do_cli_manifest_stream4(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"manifest",
"--verify-cap",
self.root_uri])
def _check((out,err)):
d = self.do_cli("manifest", "--verify-cap", self.root_uri)
def _check((rc,out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
@ -826,12 +792,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def do_cli_manifest_stream5(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"manifest",
"--repair-cap",
self.root_uri])
def _check((out,err)):
d = self.do_cli("manifest", "--repair-cap", self.root_uri)
def _check((rc,out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
@ -842,11 +804,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def do_cli_stats1(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"stats",
self.root_uri])
def _check3((out,err)):
d = self.do_cli("stats", self.root_uri)
def _check3((rc,out,err)):
lines = [l.strip() for l in out.split("\n") if l]
self.failUnless("count-immutable-files: 1" in lines)
self.failUnless("count-mutable-files: 1" in lines)
@ -862,12 +821,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def do_cli_stats2(self):
basedir = self.get_clientdir(0)
d = self._run_cli(["--node-directory", basedir,
"stats",
"--raw",
self.root_uri])
def _check4((out,err)):
d = self.do_cli("stats", "--raw", self.root_uri)
def _check4((rc,out,err)):
data = simplejson.loads(out)
self.failUnlessEqual(data["count-immutable-files"], 1)
self.failUnlessEqual(data["count-immutable-files"], 1)
@ -983,20 +938,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
return d
def _run_cli(self, argv):
stdout, stderr = StringIO(), StringIO()
# this can only do synchronous operations
assert argv[0] == "debug"
runner.runner(argv, run_by_human=False, stdout=stdout, stderr=stderr)
return stdout.getvalue()
def _delete_some_shares(self, node):
self.delete_shares_numbered(node.get_uri(), [0,1])
@defer.inlineCallbacks
def _corrupt_some_shares(self, node):
for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
if shnum in (0,1):
self._run_cli(["debug", "corrupt-share", sharefile])
yield run_cli("debug", "corrupt-share", sharefile)
def _delete_most_shares(self, node):
self.delete_shares_numbered(node.get_uri(), range(1,10))

View File

@ -1,5 +1,4 @@
import os.path, re, sys, subprocess
from cStringIO import StringIO
from twisted.trial import unittest
@ -15,6 +14,7 @@ from allmydata.client import Client
from allmydata.test import common_util
import allmydata
from allmydata import __appname__
from .common_util import run_cli
timeout = 240
@ -180,11 +180,7 @@ class CreateNode(unittest.TestCase):
fileutil.make_dirs(basedir)
return basedir
def run_tahoe(self, argv):
out,err = StringIO(), StringIO()
rc = runner.runner(argv, stdout=out, stderr=err)
return rc, out.getvalue(), err.getvalue()
@inlineCallbacks
def do_create(self, kind, *args):
basedir = self.workdir("test_" + kind)
command = "create-" + kind
@ -193,7 +189,7 @@ class CreateNode(unittest.TestCase):
n1 = os.path.join(basedir, command + "-n1")
argv = ["--quiet", command, "--basedir", n1] + list(args)
rc, out, err = self.run_tahoe(argv)
rc, out, err = yield run_cli(*argv)
self.failUnlessEqual(err, "")
self.failUnlessEqual(out, "")
self.failUnlessEqual(rc, 0)
@ -213,7 +209,7 @@ class CreateNode(unittest.TestCase):
self.failUnless("\nreserved_space = 1G\n" in content)
# creating the node a second time should be rejected
rc, out, err = self.run_tahoe(argv)
rc, out, err = yield run_cli(*argv)
self.failIfEqual(rc, 0, str((out, err, rc)))
self.failUnlessEqual(out, "")
self.failUnless("is not empty." in err)
@ -226,7 +222,7 @@ class CreateNode(unittest.TestCase):
# test that the non --basedir form works too
n2 = os.path.join(basedir, command + "-n2")
argv = ["--quiet", command] + list(args) + [n2]
rc, out, err = self.run_tahoe(argv)
rc, out, err = yield run_cli(*argv)
self.failUnlessEqual(err, "")
self.failUnlessEqual(out, "")
self.failUnlessEqual(rc, 0)
@ -236,7 +232,7 @@ class CreateNode(unittest.TestCase):
# test the --node-directory form
n3 = os.path.join(basedir, command + "-n3")
argv = ["--quiet", "--node-directory", n3, command] + list(args)
rc, out, err = self.run_tahoe(argv)
rc, out, err = yield run_cli(*argv)
self.failUnlessEqual(err, "")
self.failUnlessEqual(out, "")
self.failUnlessEqual(rc, 0)
@ -247,7 +243,7 @@ class CreateNode(unittest.TestCase):
# test that the output (without --quiet) includes the base directory
n4 = os.path.join(basedir, command + "-n4")
argv = [command] + list(args) + [n4]
rc, out, err = self.run_tahoe(argv)
rc, out, err = yield run_cli(*argv)
self.failUnlessEqual(err, "")
self.failUnlessIn(" created in ", out)
self.failUnlessIn(n4, out)

View File

@ -1,10 +1,8 @@
import os, re, sys, time, simplejson
from cStringIO import StringIO
from twisted.trial import unittest
from twisted.internet import defer
from twisted.internet import threads # CLI tests use deferToThread
from twisted.application import service
import allmydata
@ -20,7 +18,6 @@ from allmydata.util import log, base32
from allmydata.util.encodingutil import quote_output, unicode_to_argv
from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.util.consumer import MemoryConsumer, download_to_data
from allmydata.scripts import runner
from allmydata.stats import StatsGathererService
from allmydata.interfaces import IDirectoryNode, IFileNode, \
NoSuchChildError, NoSharesError
@ -39,6 +36,7 @@ from .common import TEST_RSA_KEY_SIZE
# TODO: move this to common or common_util
from allmydata.test.test_runner import RunBinTahoeMixin
from . import common_util as testutil
from .common_util import run_cli
LARGE_DATA = """
This is some data to publish to the remote grid.., which needs to be large
@ -1064,6 +1062,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d1
d.addCallback(_create_mutable)
@defer.inlineCallbacks
def _test_debug(res):
# find a share. It is important to run this while there is only
# one slot in the grid.
@ -1073,11 +1072,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
% filename)
log.msg(" for clients[%d]" % client_num)
out,err = StringIO(), StringIO()
rc = runner.runner(["debug", "dump-share", "--offsets",
filename],
stdout=out, stderr=err)
output = out.getvalue()
rc,output,err = yield run_cli("debug", "dump-share", "--offsets",
filename)
self.failUnlessEqual(rc, 0)
try:
self.failUnless("Mutable slot found:\n" in output)
@ -1862,6 +1858,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
return d
@defer.inlineCallbacks
def _test_runner(self, res):
# exercise some of the diagnostic tools in runner.py
@ -1887,11 +1884,8 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
% self.basedir)
log.msg("test_system.SystemTest._test_runner using %r" % filename)
out,err = StringIO(), StringIO()
rc = runner.runner(["debug", "dump-share", "--offsets",
unicode_to_argv(filename)],
stdout=out, stderr=err)
output = out.getvalue()
rc,output,err = yield run_cli("debug", "dump-share", "--offsets",
unicode_to_argv(filename))
self.failUnlessEqual(rc, 0)
# we only upload a single file, so we can assert some things about
@ -1917,23 +1911,18 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
sharedir, shnum = os.path.split(filename)
storagedir, storage_index_s = os.path.split(sharedir)
storage_index_s = str(storage_index_s)
out,err = StringIO(), StringIO()
nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
cmd = ["debug", "find-shares", storage_index_s] + nodedirs
rc = runner.runner(cmd, stdout=out, stderr=err)
rc,out,err = yield run_cli("debug", "find-shares", storage_index_s,
*nodedirs)
self.failUnlessEqual(rc, 0)
out.seek(0)
sharefiles = [sfn.strip() for sfn in out.readlines()]
sharefiles = [sfn.strip() for sfn in out.splitlines()]
self.failUnlessEqual(len(sharefiles), 10)
# also exercise the 'catalog-shares' tool
out,err = StringIO(), StringIO()
nodedirs = [self.getdir("client%d" % i) for i in range(self.numclients)]
cmd = ["debug", "catalog-shares"] + nodedirs
rc = runner.runner(cmd, stdout=out, stderr=err)
rc,out,err = yield run_cli("debug", "catalog-shares", *nodedirs)
self.failUnlessEqual(rc, 0)
out.seek(0)
descriptions = [sfn.strip() for sfn in out.readlines()]
descriptions = [sfn.strip() for sfn in out.splitlines()]
self.failUnlessEqual(len(descriptions), 30)
matching = [line
for line in descriptions
@ -1981,10 +1970,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
f.write(private_uri)
f.close()
@defer.inlineCallbacks
def run(ignored, verb, *args, **kwargs):
stdin = kwargs.get("stdin", "")
newargs = nodeargs + [verb] + list(args)
return self._run_cli(newargs, stdin=stdin)
rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
defer.returnValue((out,err))
def _check_ls((out,err), expected_children, unexpected_children=[]):
self.failUnlessEqual(err, "")
@ -2345,17 +2334,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
d.addCallback(_check_ls)
return d
def _run_cli(self, argv, stdin=""):
#print "CLI:", argv
stdout, stderr = StringIO(), StringIO()
d = threads.deferToThread(runner.runner, argv, run_by_human=False,
stdin=StringIO(stdin),
stdout=stdout, stderr=stderr)
def _done(res):
return stdout.getvalue(), stderr.getvalue()
d.addCallback(_done)
return d
def _test_checker(self, res):
ut = upload.Data("too big to be literal" * 200, convergence=None)
d = self._personal_node.add_file(u"big file", ut)