ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-19 13:07:56 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

#509 CLI: add 'tahoe manifest --stream'

Browse Source
This commit is contained in:
Brian Warner 2009-01-23 16:33:21 -07:00
parent 5a0e3f3a40
commit 6a04103cc2
3 changed files with 175 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/allmydata/scripts/cli.py
View File

@ -200,7 +200,10 @@ class WebopenOptions(VDriveOptions):
class ManifestOptions(VDriveOptions):
optFlags = [
("storage-index", "s", "Only print storage index strings, not pathname+cap"),
("verify-cap", None, "Only print verifycap, not pathname+cap"),
("repair-cap", None, "Only print repaircap, not pathname+cap"),
("raw", "r", "Display raw JSON data instead of parsed"),
("stream", None, "Stream lines of JSON to stdout. With --storage-index or --verify-cap, stream those strings to stdout."),
]
def parseArgs(self, where=''):
self.where = where

74
src/allmydata/scripts/tahoe_manifest.py
View File

@ -1,8 +1,77 @@
import urllib, simplejson
from twisted.protocols.basic import LineOnlyReceiver
from allmydata.util import base32
from allmydata.util.abbreviate import abbreviate_space_both
from allmydata import uri
from allmydata.scripts.slow_operation import SlowOperationRunner
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path
from allmydata.scripts.common_http import do_http
class FakeTransport:
disconnecting = False
class ManifestStreamer(LineOnlyReceiver):
delimiter = "\n"
def __init__(self):
self.transport = FakeTransport()
def run(self, options):
stdout = options.stdout
stderr = options.stderr
self.options = options
nodeurl = options['node-url']
if not nodeurl.endswith("/"):
nodeurl += "/"
self.nodeurl = nodeurl
where = options.where
rootcap, path = get_alias(options.aliases, where, DEFAULT_ALIAS)
if path == '/':
path = ''
url = nodeurl + "uri/%s" % urllib.quote(rootcap)
if path:
url += "/" + escape_path(path)
# todo: should it end with a slash?
url += "?t=stream-manifest"
resp = do_http("POST", url)
if resp.status not in (200, 302):
print >>stderr, "ERROR", resp.status, resp.reason, resp.read()
return 1
#print "RESP", dir(resp)
# use Twisted to split this into lines
while True:
chunk = resp.read(100)
if not chunk:
break
if self.options["raw"]:
stdout.write(chunk)
else:
self.dataReceived(chunk)
return 0
def lineReceived(self, line):
d = simplejson.loads(line)
stdout = self.options.stdout
if d["type"] in ("file", "directory"):
if self.options["storage-index"]:
si = d["storage-index"]
if si:
print >>stdout, si
elif self.options["verify-cap"]:
vc = d["verifycap"]
if vc:
print >>stdout, vc
elif self.options["repair-cap"]:
vc = d["repaircap"]
if vc:
print >>stdout, vc
else:
try:
print >>stdout, d["cap"], "/".join(d["path"])
except UnicodeEncodeError:
print >>stdout, d["cap"], "/".join([p.encode("utf-8")
for p in d["path"]])
class ManifestGrabber(SlowOperationRunner):
@ -27,7 +96,10 @@ class ManifestGrabber(SlowOperationRunner):
for p in path])
def manifest(options):
return ManifestGrabber().run(options)
if options["stream"]:
return ManifestStreamer().run(options)
else:
return ManifestGrabber().run(options)
class StatsGrabber(SlowOperationRunner):

106
src/allmydata/test/test_system.py
View File

@ -2481,6 +2481,11 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d.addCallback(lambda ign: self.do_cli_manifest1())
d.addCallback(lambda ign: self.do_cli_manifest2())
d.addCallback(lambda ign: self.do_cli_manifest3())
d.addCallback(lambda ign: self.do_cli_manifest_stream1())
d.addCallback(lambda ign: self.do_cli_manifest_stream2())
d.addCallback(lambda ign: self.do_cli_manifest_stream3())
d.addCallback(lambda ign: self.do_cli_manifest_stream4())
d.addCallback(lambda ign: self.do_cli_manifest_stream5())
d.addCallback(lambda ign: self.do_cli_stats1())
d.addCallback(lambda ign: self.do_cli_stats2())
return d
@ -2516,16 +2521,19 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--storage-index", self.root_uri])
def _check2((out,err)):
def _check((out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
d.addCallback(_check2)
self._check_manifest_storage_index(out)
d.addCallback(_check)
return d
def _check_manifest_storage_index(self, out):
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.large.get_storage_index()) in lines)
def do_cli_manifest3(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
@ -2550,6 +2558,90 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d.addCallback(_check2r)
return d
def do_cli_manifest_stream1(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--stream",
self.root_uri])
def _check((out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 5)
caps = {}
for l in lines:
try:
cap, path = l.split(None, 1)
except ValueError:
cap = l.strip()
path = ""
caps[cap] = path
self.failUnless(self.root.get_uri() in caps)
self.failUnlessEqual(caps[self.root.get_uri()], "")
self.failUnlessEqual(caps[self.mutable.get_uri()], "mutable")
self.failUnlessEqual(caps[self.large.get_uri()], "large")
self.failUnlessEqual(caps[self.small.get_uri()], "small")
self.failUnlessEqual(caps[self.small2.get_uri()], "small2")
d.addCallback(_check)
return d
def do_cli_manifest_stream2(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--stream", "--raw",
self.root_uri])
def _check((out,err)):
self.failUnlessEqual(err, "")
# this should be the same as the POST t=stream-manifest output
self._check_streamed_manifest(out)
d.addCallback(_check)
return d
def do_cli_manifest_stream3(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--stream", "--storage-index",
self.root_uri])
def _check((out,err)):
self.failUnlessEqual(err, "")
self._check_manifest_storage_index(out)
d.addCallback(_check)
return d
def do_cli_manifest_stream4(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--stream", "--verify-cap",
self.root_uri])
def _check((out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_verify_cap().to_string() in lines)
self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
self.failUnless(self.large.get_verify_cap().to_string() in lines)
d.addCallback(_check)
return d
def do_cli_manifest_stream5(self):
basedir = self.getdir("client0")
d = self._run_cli(["manifest",
"--node-directory", basedir,
"--stream", "--repair-cap",
self.root_uri])
def _check((out,err)):
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_repair_cap().to_string() in lines)
self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
self.failUnless(self.large.get_repair_cap().to_string() in lines)
d.addCallback(_check)
return d
def do_cli_stats1(self):
basedir = self.getdir("client0")
d = self._run_cli(["stats",