From 20084506b330e602c1a2a28dbf989773f3d3d783 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 23 May 2018 13:59:42 -0400 Subject: [PATCH] Handle all the status cases --- src/allmydata/scripts/tahoe_status.py | 236 +++++++++++++++++--------- src/allmydata/test/cli/test_status.py | 48 +++--- src/allmydata/test/status.py | 16 ++ src/allmydata/test/web/test_web.py | 13 +- src/allmydata/web/status.py | 76 ++++----- 5 files changed, 241 insertions(+), 148 deletions(-) create mode 100644 src/allmydata/test/status.py diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index f66fdf9c5..405a8c730 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -83,6 +83,163 @@ def pretty_progress(percent, size=10, ascii=False): curr = int(curr) return '%s%s%s' % ((block_chr * curr), part, (' ' * (size - curr - 1))) +OP_MAP = { + 'upload': ' put ', + 'download': ' get ', + 'retrieve': 'retr ', + 'publish': ' pub ', + 'mapupdate': 'mapup', + 'unknown': ' ??? ', +} + +def _render_active_upload(op): + total = ( + op['progress-hash'] + + op['progress-ciphertext'] + + op['progress-encode-push'] + ) / 3.0 * 100.0 + return { + u"op_type": u" put ", + u"total": "{:3.0f}".format(total), + u"progress_bar": u"{}".format(pretty_progress(total, size=15)), + u"storage-index-string": op["storage-index-string"], + u"status": op["status"], + } + +def _render_active_download(op): + return { + u"op_type": u" get ", + u"total": op["progress"], + u"progress_bar": u"{}".format(pretty_progress(op['progress'] * 100.0, size=15)), + u"storage-index-string": op["storage-index-string"], + u"status": op["status"], + } + +def _render_active_generic(op): + return { + u"op_type": OP_MAP[op["type"]], + u"progress_bar": u"", + u"total": u"???", + u"storage-index-string": op["storage-index-string"], + u"status": op["status"], + } + +active_renderers = { + "upload": _render_active_upload, + "download": _render_active_download, + "publish": _render_active_generic, + "retrieve": _render_active_generic, + "mapupdate": _render_active_generic, + "unknown": _render_active_generic, +} + + +def render_active(stdout, status_data): + active = status_data.get('active', None) + if not active: + print(u"No active operations.", file=stdout) + return + + header = u"\u2553 {:<5} \u2565 {:<26} \u2565 {:<22} \u2565 {}".format( + "type", + "storage index", + "progress", + "status message", + ) + header_bar = u"\u255f\u2500{}\u2500\u256b\u2500{}\u2500\u256b\u2500{}\u2500\u256b\u2500{}".format( + u'\u2500' * 5, + u'\u2500' * 26, + u'\u2500' * 22, + u'\u2500' * 20, + ) + line_template = ( + u"\u2551 {op_type} " + u"\u2551 {storage-index-string} " + u"\u2551 {progress_bar:15} " + u"({total}%) " + u"\u2551 {status}" + ) + footer_bar = u"\u2559\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}".format( + u'\u2500' * 5, + u'\u2500' * 26, + u'\u2500' * 22, + u'\u2500' * 20, + ) + print(u"Active operations:", file=stdout) + print(header, file=stdout) + print(header_bar, file=stdout) + for op in active: + print(line_template.format( + **active_renderers[op["type"]](op) + )) + print(footer_bar, file=stdout) + +def _render_recent_generic(op): + return { + u"op_type": OP_MAP[op["type"]], + u"storage-index-string": op["storage-index-string"], + u"nice_size": abbreviate_space(op["total-size"]), + u"status": op["status"], + } + +def _render_recent_mapupdate(op): + return { + u"op_type": u"mapup", + u"storage-index-string": op["storage-index-string"], + u"nice_size": op["mode"], + u"status": op["status"], + } + +recent_renderers = { + "upload": _render_recent_generic, + "download": _render_recent_generic, + "publish": _render_recent_generic, + "retrieve": _render_recent_generic, + "mapupdate": _render_recent_mapupdate, + "unknown": _render_recent_generic, +} + +def render_recent(verbose, stdout, status_data): + recent = status_data.get('recent', None) + if not recent: + print(u"No recent operations.", file=stdout) + + header = u"\u2553 {:<5} \u2565 {:<26} \u2565 {:<10} \u2565 {}".format( + "type", + "storage index", + "size", + "status message", + ) + line_template = ( + u"\u2551 {op_type} " + u"\u2551 {storage-index-string} " + u"\u2551 {nice_size:<10} " + u"\u2551 {status}" + ) + footer = u"\u2559\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}".format( + u'\u2500' * 5, + u'\u2500' * 26, + u'\u2500' * 10, + u'\u2500' * 20, + ) + non_verbose_ops = ('upload', 'download') + recent = [op for op in status_data['recent'] if op['type'] in non_verbose_ops] + print(u"\nRecent operations:", file=stdout) + if len(recent) or verbose: + print(header, file=stdout) + + ops_to_show = status_data['recent'] if verbose else recent + for op in ops_to_show: + print(line_template.format( + **recent_renderers[op["type"]](op) + )) + if len(recent) or verbose: + print(footer, file=stdout) + + skipped = len(status_data['recent']) - len(ops_to_show) + if not verbose and skipped: + print(u" Skipped {} non-upload/download operations; use --verbose to see".format(skipped), file=stdout) + def do_status(options): nodedir = options["node-directory"] @@ -125,83 +282,8 @@ def do_status(options): print(u" downloaded {} in {} files".format(abbreviate_space(downloaded_bytes), downloaded_files), file=options.stdout) print(u"", file=options.stdout) - if status_data.get('active', None): - print(u"Active operations:", file=options.stdout) - print( - u"\u2553 {:<5} \u2565 {:<26} \u2565 {:<22} \u2565 {}".format( - "type", - "storage index", - "progress", - "status message", - ), file=options.stdout - ) - print(u"\u255f\u2500{}\u2500\u256b\u2500{}\u2500\u256b\u2500{}\u2500\u256b\u2500{}".format(u'\u2500' * 5, u'\u2500' * 26, u'\u2500' * 22, u'\u2500' * 20), file=options.stdout) - for op in status_data['active']: - if 'progress-hash' in op: - op_type = ' put ' - total = (op['progress-hash'] + op['progress-ciphertext'] + op['progress-encode-push']) / 3.0 - progress_bar = u"{}".format(pretty_progress(total * 100.0, size=15)) - else: - op_type = ' get ' - total = op['progress'] - progress_bar = u"{}".format(pretty_progress(op['progress'] * 100.0, size=15)) - print( - u"\u2551 {op_type} \u2551 {storage-index-string} \u2551 {progress_bar} ({total:3}%) \u2551 {status}".format( - op_type=op_type, - progress_bar=progress_bar, - total=int(total * 100.0), - **op - ), file=options.stdout - ) - - print(u"\u2559\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}".format(u'\u2500' * 5, u'\u2500' * 26, u'\u2500' * 22, u'\u2500' * 20), file=options.stdout) - else: - print(u"No active operations.", file=options.stdout) - - if status_data.get('recent', None): - non_verbose_ops = ('upload', 'download') - recent = [op for op in status_data['recent'] if op['type'] in non_verbose_ops] - print(u"\nRecent operations:", file=options.stdout) - if len(recent) or options['verbose']: - print( - u"\u2553 {:<5} \u2565 {:<26} \u2565 {:<10} \u2565 {}".format( - "type", - "storage index", - "size", - "status message", - ), file=options.stdout - ) - - op_map = { - 'upload': ' put ', - 'download': ' get ', - 'retrieve': 'retr ', - 'publish': ' pub ', - 'mapupdate': 'mapup', - } - - ops_to_show = status_data['recent'] if options['verbose'] else recent - for op in ops_to_show: - op_type = op_map[op.get('type', None)] - if op['type'] == 'mapupdate': - nice_size = op['mode'] - else: - nice_size = abbreviate_space(op['total-size']) - print( - u"\u2551 {op_type} \u2551 {storage-index-string} \u2551 {nice_size:<10} \u2551 {status}".format( - op_type=op_type, - nice_size=nice_size, - **op - ), file=options.stdout - ) - - if len(recent) or options['verbose']: - print(u"\u2559\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}\u2500\u2568\u2500{}".format(u'\u2500' * 5, u'\u2500' * 26, u'\u2500' * 10, u'\u2500' * 20), file=options.stdout) - skipped = len(status_data['recent']) - len(ops_to_show) - if not options['verbose'] and skipped: - print(u" Skipped {} non-upload/download operations; use --verbose to see".format(skipped), file=options.stdout) - else: - print(u"No recent operations.", file=options.stdout) + render_active(options.stdout, status_data) + render_recent(options['verbose'], options.stdout, status_data) # open question: should we return non-zero if there were no # operations at all to display? diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 0ae420a7c..e6ee4be37 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -15,9 +15,17 @@ from allmydata.scripts.tahoe_status import _get_json_for_fragment from allmydata.scripts.tahoe_status import _get_json_for_cap from allmydata.scripts.tahoe_status import pretty_progress from allmydata.scripts.tahoe_status import do_status +from allmydata.web.status import marshal_json + +from allmydata.immutable.upload import UploadStatus +from allmydata.immutable.downloader.status import DownloadStatus +from allmydata.mutable.publish import PublishStatus +from allmydata.mutable.retrieve import RetrieveStatus +from allmydata.mutable.servermap import UpdateStatus from ..no_network import GridTestMixin from ..common_web import do_http +from ..status import FakeStatus from .common import CLITestMixin @@ -135,30 +143,26 @@ class CommandStatus(unittest.TestCase): @mock.patch('allmydata.scripts.tahoe_status.do_http') @mock.patch('sys.stdout', StringIO()) def test_simple(self, http): + recent_items = active_items = [ + UploadStatus(), + DownloadStatus("abcd", 12345), + PublishStatus(), + RetrieveStatus(), + UpdateStatus(), + FakeStatus(), + ] values = [ StringIO(json.dumps({ - "active": [ - { - "progress": 0.5, - "storage-index-string": "index0", - "status": "foo", - }, - { - "progress-hash": 1.0, - "progress-ciphertext": 1.0, - "progress-encode-push": 0.5, - "storage-index-string": "index1", - "status": "bar", - } - ], - "recent": [ - { - "type": "download", - "total-size": 12345, - "storage-index-string": "index1", - "status": "bar", - }, - ] + "active": list( + marshal_json(item) + for item + in active_items + ), + "recent": list( + marshal_json(item) + for item + in recent_items + ), })), StringIO(json.dumps({ "counters": { diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py new file mode 100644 index 000000000..44f2123f9 --- /dev/null +++ b/src/allmydata/test/status.py @@ -0,0 +1,16 @@ + +class FakeStatus(object): + def __init__(self): + self.status = [] + + def setServiceParent(self, p): + pass + + def get_status(self): + return self.status + + def get_storage_index(self): + return None + + def get_size(self): + return None diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index e7c309d99..7631327c9 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -50,6 +50,8 @@ from ..common_web import ( ) from allmydata.client import _Client, SecretHolder from .common import unknown_rwcap, unknown_rocap, unknown_immcap, FAVICON_MARKUP +from ..status import FakeStatus + # create a fake uploader/downloader, and a couple of fake dirnodes, then # create a webserver that works against them @@ -111,17 +113,6 @@ class FakeUploader(service.Service): return (self.helper_furl, self.helper_connected) -class FakeStatus(object): - def __init__(self): - self.status = [] - - def setServiceParent(self, p): - pass - - def get_status(self): - return self.status - - def create_test_queued_item(relpath_u, history=[]): progress = mock.Mock() progress.progress = 100.0 diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 31831d8a2..5433d6c6d 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -958,6 +958,42 @@ class MapupdateStatusPage(rend.Page, RateAndTimeMixin): return T.li["Per-Server Response Times: ", l] +def marshal_json(s): + # common item data + item = { + "storage-index-string": base32.b2a_or_none(s.get_storage_index()), + "total-size": s.get_size(), + "status": s.get_status(), + } + + # type-specific item date + if IUploadStatus.providedBy(s): + h, c, e = s.get_progress() + item["type"] = "upload" + item["progress-hash"] = h + item["progress-ciphertext"] = c + item["progress-encode-push"] = e + + elif IDownloadStatus.providedBy(s): + item["type"] = "download" + item["progress"] = s.get_progress() + + elif IPublishStatus.providedBy(s): + item["type"] = "publish" + + elif IRetrieveStatus.providedBy(s): + item["type"] = "retrieve" + + elif IServermapUpdaterStatus.providedBy(s): + item["type"] = "mapupdate" + item["mode"] = s.get_mode() + + else: + item["type"] = "unknown" + item["class"] = s.__class__.__name__ + + return item + class Status(MultiFormatPage): docFactory = getxmlfile("status.xhtml") @@ -974,47 +1010,11 @@ class Status(MultiFormatPage): data["active"] = active = [] data["recent"] = recent = [] - def _marshal_json(s): - # common item data - item = { - "storage-index-string": base32.b2a_or_none(s.get_storage_index()), - "total-size": s.get_size(), - "status": s.get_status(), - } - - # type-specific item date - if IUploadStatus.providedBy(s): - h, c, e = s.get_progress() - item["type"] = "upload" - item["progress-hash"] = h - item["progress-ciphertext"] = c - item["progress-encode-push"] = e - - elif IDownloadStatus.providedBy(s): - item["type"] = "download" - item["progress"] = s.get_progress() - - elif IPublishStatus.providedBy(s): - item["type"] = "publish" - - elif IRetrieveStatus.providedBy(s): - item["type"] = "retrieve" - - elif IServermapUpdaterStatus.providedBy(s): - item["type"] = "mapupdate" - item["mode"] = s.get_mode() - - else: - item["type"] = "unknown" - item["class"] = s.__class__.__name__ - - return item - for s in self._get_active_operations(): - active.append(_marshal_json(s)) + active.append(marshal_json(s)) for s in self._get_recent_operations(): - recent.append(_marshal_json(s)) + recent.append(marshal_json(s)) return json.dumps(data, indent=1) + "\n"