mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-06-18 23:38:18 +00:00
Another checker script.
This commit is contained in:
197
misc/checkers/check_grid.py
Normal file
197
misc/checkers/check_grid.py
Normal file
@ -0,0 +1,197 @@
|
||||
from __future__ import print_function
|
||||
|
||||
"""
|
||||
Test an existing Tahoe grid, both to see if the grid is still running and to
|
||||
see if the client is still compatible with it. This script is suitable for
|
||||
running from a periodic monitoring script, perhaps by an hourly cronjob.
|
||||
|
||||
This script uses a pre-established client node (configured to connect to the
|
||||
grid being tested) and a pre-established directory (stored as the 'testgrid:'
|
||||
alias in that client node's aliases file). It then performs a number of
|
||||
uploads and downloads to exercise compatibility in various directions (new
|
||||
client vs old data). All operations are performed by invoking various CLI
|
||||
commands through bin/tahoe . The script must be given two arguments: the
|
||||
client node directory, and the location of the bin/tahoe executable. Note
|
||||
that this script does not import anything from tahoe directly, so it doesn't
|
||||
matter what its PYTHONPATH is, as long as the bin/tahoe that it uses is
|
||||
functional.
|
||||
|
||||
This script expects the client node to be running already.
|
||||
|
||||
To set up the client node, do the following:
|
||||
|
||||
tahoe create-client --introducer=INTRODUCER_FURL DIR
|
||||
tahoe run DIR
|
||||
tahoe -d DIR create-alias testgrid
|
||||
# pick a 10kB-ish test file, compute its md5sum
|
||||
tahoe -d DIR put FILE testgrid:old.MD5SUM
|
||||
tahoe -d DIR put FILE testgrid:recent.MD5SUM
|
||||
tahoe -d DIR put FILE testgrid:recentdir/recent.MD5SUM
|
||||
echo "" | tahoe -d DIR put --mutable - testgrid:log
|
||||
echo "" | tahoe -d DIR put --mutable - testgrid:recentlog
|
||||
|
||||
This script will perform the following steps (the kind of compatibility that
|
||||
is being tested is in [brackets]):
|
||||
|
||||
read old.* and check the md5sums [confirm that new code can read old files]
|
||||
read all recent.* files and check md5sums [read recent files]
|
||||
delete all recent.* files and verify they're gone [modify an old directory]
|
||||
read recentdir/recent.* files and check [read recent directory]
|
||||
delete recentdir/recent.* and verify [modify recent directory]
|
||||
delete recentdir and verify (keep the directory from growing unboundedly)
|
||||
mkdir recentdir
|
||||
upload random 10kB file to recentdir/recent.MD5SUM (prepare for next time)
|
||||
upload random 10kB file to recent.MD5SUM [new code can upload to old servers]
|
||||
append one-line timestamp to log [read/write old mutable files]
|
||||
append one-line timestamp to recentlog [read/write recent mutable files]
|
||||
delete recentlog
|
||||
upload small header to new mutable recentlog [create mutable files]
|
||||
|
||||
This script will also keep track of speeds and latencies and will write them
|
||||
in a machine-readable logfile.
|
||||
"""
|
||||
|
||||
import time, subprocess, md5, os.path, random
|
||||
from twisted.python import usage
|
||||
|
||||
class GridTesterOptions(usage.Options):
|
||||
|
||||
optFlags = [
|
||||
("no", "n", "Dry run: do not run any commands, just print them."),
|
||||
]
|
||||
|
||||
def parseArgs(self, nodedir, tahoe):
|
||||
# Note: does not support Unicode arguments.
|
||||
self.nodedir = os.path.expanduser(nodedir)
|
||||
self.tahoe = os.path.abspath(os.path.expanduser(tahoe))
|
||||
|
||||
class CommandFailed(Exception):
|
||||
pass
|
||||
|
||||
class GridTester(object):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.tahoe = config.tahoe
|
||||
self.nodedir = config.nodedir
|
||||
|
||||
def command(self, *cmd, **kwargs):
|
||||
expected_rc = kwargs.get("expected_rc", 0)
|
||||
stdin = kwargs.get("stdin", None)
|
||||
if self.config["no"]:
|
||||
return
|
||||
if stdin is not None:
|
||||
p = subprocess.Popen(cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
(stdout,stderr) = p.communicate(stdin)
|
||||
else:
|
||||
p = subprocess.Popen(cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
(stdout,stderr) = p.communicate()
|
||||
rc = p.returncode
|
||||
if expected_rc != None and rc != expected_rc:
|
||||
if stderr:
|
||||
print("STDERR:")
|
||||
print(stderr)
|
||||
raise CommandFailed("command '%s' failed: rc=%d" % (cmd, rc))
|
||||
return stdout, stderr
|
||||
|
||||
def cli(self, cmd, *args, **kwargs):
|
||||
print("tahoe", cmd, " ".join(args))
|
||||
stdout, stderr = self.command(self.tahoe, "-d", self.nodedir, cmd,
|
||||
*args, **kwargs)
|
||||
if not kwargs.get("ignore_stderr", False) and stderr != "":
|
||||
raise CommandFailed("command '%s' had stderr: %s" % (" ".join(args),
|
||||
stderr))
|
||||
return stdout
|
||||
|
||||
def read_and_check(self, f):
|
||||
expected_md5_s = f[f.find(".")+1:]
|
||||
out = self.cli("get", "testgrid:" + f)
|
||||
got_md5_s = md5.new(out).hexdigest()
|
||||
if got_md5_s != expected_md5_s:
|
||||
raise CommandFailed("%s had md5sum of %s" % (f, got_md5_s))
|
||||
|
||||
def delete_and_check(self, dirname, f):
|
||||
oldfiles = self.listdir(dirname)
|
||||
if dirname:
|
||||
absfilename = "testgrid:" + dirname + "/" + f
|
||||
else:
|
||||
absfilename = "testgrid:" + f
|
||||
if f not in oldfiles:
|
||||
raise CommandFailed("um, '%s' was supposed to already be in %s"
|
||||
% (f, dirname))
|
||||
self.cli("unlink", absfilename)
|
||||
newfiles = self.listdir(dirname)
|
||||
if f in newfiles:
|
||||
raise CommandFailed("failed to remove '%s' from %s" % (f, dirname))
|
||||
|
||||
def listdir(self, dirname):
|
||||
out = self.cli("ls", "testgrid:"+dirname).strip().split("\n")
|
||||
files = [f.strip() for f in out]
|
||||
print(" ", files)
|
||||
return files
|
||||
|
||||
def do_test(self):
|
||||
files = self.listdir("")
|
||||
for f in files:
|
||||
if f.startswith("old.") or f.startswith("recent."):
|
||||
self.read_and_check("" + f)
|
||||
for f in files:
|
||||
if f.startswith("recent."):
|
||||
self.delete_and_check("", f)
|
||||
files = self.listdir("recentdir")
|
||||
for f in files:
|
||||
if f.startswith("old.") or f.startswith("recent."):
|
||||
self.read_and_check("recentdir/" + f)
|
||||
for f in files:
|
||||
if f.startswith("recent."):
|
||||
self.delete_and_check("recentdir", f)
|
||||
self.delete_and_check("", "recentdir")
|
||||
|
||||
self.cli("mkdir", "testgrid:recentdir")
|
||||
fn, data = self.makefile("recent")
|
||||
self.put("recentdir/"+fn, data)
|
||||
files = self.listdir("recentdir")
|
||||
if fn not in files:
|
||||
raise CommandFailed("failed to put %s in recentdir/" % fn)
|
||||
fn, data = self.makefile("recent")
|
||||
self.put(fn, data)
|
||||
files = self.listdir("")
|
||||
if fn not in files:
|
||||
raise CommandFailed("failed to put %s in testgrid:" % fn)
|
||||
|
||||
self.update("log")
|
||||
self.update("recentlog")
|
||||
self.delete_and_check("", "recentlog")
|
||||
self.put_mutable("recentlog", "Recent Mutable Log Header\n\n")
|
||||
|
||||
def put(self, fn, data):
|
||||
self.cli("put", "-", "testgrid:"+fn, stdin=data, ignore_stderr=True)
|
||||
|
||||
def put_mutable(self, fn, data):
|
||||
self.cli("put", "--mutable", "-", "testgrid:"+fn,
|
||||
stdin=data, ignore_stderr=True)
|
||||
|
||||
def update(self, fn):
|
||||
old = self.cli("get", "testgrid:"+fn)
|
||||
new = old + time.ctime() + "\n"
|
||||
self.put(fn, new)
|
||||
|
||||
def makefile(self, prefix):
|
||||
size = random.randint(10001, 10100)
|
||||
data = os.urandom(size)
|
||||
md5sum = md5.new(data).hexdigest()
|
||||
fn = prefix + "." + md5sum
|
||||
return fn, data
|
||||
|
||||
def main():
|
||||
config = GridTesterOptions()
|
||||
config.parseOptions()
|
||||
gt = GridTester(config)
|
||||
gt.do_test()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Reference in New Issue
Block a user