From d142ccb159c70da4b1c1e4159f955f06b5deb7c2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 06:46:08 -0500 Subject: [PATCH 001/134] Use twisted.web.template in web/storage.py Related to ticket:3247. Nevow usage has been removed, and generated page looks the same as its former self, but tests are failing because test_storage.py assumes that we're using nevow. --- src/allmydata/web/storage.py | 258 ++++++++++++++----------- src/allmydata/web/storage_status.xhtml | 60 +++--- 2 files changed, 182 insertions(+), 136 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 79c0a8c38..3000fb70d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,10 +1,10 @@ import time, json -from nevow import rend, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile from allmydata.web.common import ( - getxmlfile, abbreviate_time, - MultiFormatPage, + MultiFormatResource ) from allmydata.util.abbreviate import abbreviate_space from allmydata.util import time_format, idlib @@ -16,91 +16,100 @@ def remove_prefix(s, prefix): return s[len(prefix):] -class StorageStatus(MultiFormatPage): - docFactory = getxmlfile("storage_status.xhtml") - # the default 'data' argument is the StorageServer instance +class StorageStatusElement(Element): + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname=""): - rend.Page.__init__(self, storage) + def __init__(self, storage, nickname): + super(StorageStatusElement, self).__init__() self.storage = storage - self.nickname = nickname + self.nick = nickname - def render_JSON(self, req): - req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), - } - return json.dumps(d, indent=1) + "\n" + @renderer + def nickname(self, req, tag): + return self.nick - def data_nickname(self, ctx, storage): - return self.nickname - def data_nodeid(self, ctx, storage): + @renderer + def nodeid(self, req, tag): return idlib.nodeid_b2a(self.storage.my_nodeid) - def render_storage_running(self, ctx, storage): - if storage: - return ctx.tag - else: - return T.h1["No Storage Server Running"] + def get_stat(self, key): + return self.storage.get_stats().get(key) - def render_bool(self, ctx, data): - return {True: "Yes", False: "No"}[bool(data)] + def str(self, tag, val): + if val is None: + return tag("?") + return tag(str(val)) - def render_abbrev_space(self, ctx, size): - if size is None: - return "?" - return abbreviate_space(size) + def abbr(self, tag, val): + if val is None: + return tag("?") + return tag(abbreviate_space(val)) - def render_space(self, ctx, size): - if size is None: - return "?" - return "%d" % size + @renderer + def disk_total(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_total")) - def data_stats(self, ctx, data): - # FYI: 'data' appears to be self, rather than the StorageServer - # object in self.original that gets passed to render_* methods. I - # still don't understand Nevow. + @renderer + def disk_total_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_total")) - # Nevow has nevow.accessors.DictionaryContainer: Any data= directive - # that appears in a context in which the current data is a dictionary - # will be looked up as keys in that dictionary. So if data_stats() - # returns a dictionary, then we can use something like this: - # - # + @renderer + def disk_used(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_used")) - # to use get_stats()["storage_server.disk_total"] . However, - # DictionaryContainer does a raw d[] instead of d.get(), so any - # missing keys will cause an error, even if the renderer can tolerate - # None values. To overcome this, we either need a dict-like object - # that always returns None for unknown keys, or we must pre-populate - # our dict with those missing keys, or we should get rid of data_ - # methods that return dicts (or find some way to override Nevow's - # handling of dictionaries). + @renderer + def disk_used_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_used")) - d = dict([ (remove_prefix(k, "storage_server."), v) - for k,v in self.storage.get_stats().items() ]) - d.setdefault("disk_total", None) - d.setdefault("disk_used", None) - d.setdefault("disk_free_for_root", None) - d.setdefault("disk_free_for_nonroot", None) - d.setdefault("reserved_space", None) - d.setdefault("disk_avail", None) - return d + @renderer + def disk_free_for_root(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) - def data_last_complete_bucket_count(self, ctx, data): + @renderer + def disk_free_for_root_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + + @renderer + def disk_free_for_nonroot(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def disk_free_for_nonroot_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + + @renderer + def reserved_space(self, req, tag): + return self.str(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def reserved_space_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + + @renderer + def disk_avail(self, req, tag): + return self.str(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def disk_avail_abbrev(self, req, tag): + return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + + @renderer + def accepting_immutable_shares(self, req, tag): + accepting = self.get_stat("storage_server.accepting_immutable_shares") + return {True: "Yes", False: "No"}[bool(accepting)] + + @renderer + def last_complete_bucket_count(self, req, tag): s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" - return count + return str(count) - def render_count_crawler_status(self, ctx, storage): + @renderer + def count_crawler_status(self, req, tag): p = self.storage.bucket_counter.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return self.format_crawler_progress(p) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] @@ -127,55 +136,51 @@ class StorageStatus(MultiFormatPage): return ["Next crawl in %s" % abbreviate_time(soon), cycletime_s] - def render_lease_expiration_enabled(self, ctx, data): + @renderer + def storage_running(self, req, tag): + if self.storage: + return tag + return tag("No Storage Server Running") + + @renderer + def lease_expiration_enabled(self, req, tag): lc = self.storage.lease_checker if lc.expiration_enabled: - return ctx.tag["Enabled: expired leases will be removed"] + return tag("Enabled: expired leases will be removed") else: - return ctx.tag["Disabled: scan-only mode, no leases will be removed"] + return tag("Disabled: scan-only mode, no leases will be removed") - def render_lease_expiration_mode(self, ctx, data): + @renderer + def lease_expiration_mode(self, req, tag): lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: - ctx.tag["Leases will expire naturally, probably 31 days after " - "creation or renewal."] + tag("Leases will expire naturally, probably 31 days after " + "creation or renewal.") else: - ctx.tag["Leases created or last renewed more than %s ago " - "will be considered expired." - % abbreviate_time(lc.override_lease_duration)] + tag("Leases created or last renewed more than %s ago " + "will be considered expired." + % abbreviate_time(lc.override_lease_duration)) else: assert lc.mode == "cutoff-date" localizedutcdate = time.strftime("%d-%b-%Y", time.gmtime(lc.cutoff_date)) isoutcdate = time_format.iso_utc_date(lc.cutoff_date) - ctx.tag["Leases created or last renewed before %s (%s) UTC " - "will be considered expired." % (isoutcdate, localizedutcdate, )] + tag("Leases created or last renewed before %s (%s) UTC " + "will be considered expired." + % (isoutcdate, localizedutcdate, )) if len(lc.mode) > 2: - ctx.tag[" The following sharetypes will be expired: ", - " ".join(sorted(lc.sharetypes_to_expire)), "."] - return ctx.tag + tag(" The following sharetypes will be expired: ", + " ".join(sorted(lc.sharetypes_to_expire)), ".") + return tag - def format_recovered(self, sr, a): - def maybe(d): - if d is None: - return "?" - return "%d" % d - return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ - (maybe(sr["%s-shares" % a]), - maybe(sr["%s-buckets" % a]), - maybe(sr["%s-buckets-mutable" % a]), - maybe(sr["%s-buckets-immutable" % a]), - abbreviate_space(sr["%s-diskbytes" % a]), - abbreviate_space(sr["%s-diskbytes-mutable" % a]), - abbreviate_space(sr["%s-diskbytes-immutable" % a]), - ) - - def render_lease_current_cycle_progress(self, ctx, data): + @renderer + def lease_current_cycle_progress(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() - return ctx.tag[self.format_crawler_progress(p)] + return tag(self.format_crawler_progress(p)) - def render_lease_current_cycle_results(self, ctx, data): + @renderer + def lease_current_cycle_results(self, req, tag): lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -229,10 +234,10 @@ class StorageStatus(MultiFormatPage): T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] ]]]) + return tag("Current cycle:", p) - return ctx.tag["Current cycle:", p] - - def render_lease_last_cycle_results(self, ctx, data): + @renderer + def lease_last_cycle_results(self, req, tag): lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: @@ -240,15 +245,15 @@ class StorageStatus(MultiFormatPage): last = h[max(h.keys())] start, end = last["cycle-start-finish-times"] - ctx.tag["Last complete cycle (which took %s and finished %s ago)" - " recovered: " % (abbreviate_time(end-start), - abbreviate_time(time.time() - end)), - self.format_recovered(last["space-recovered"], "actual") - ] + tag("Last complete cycle (which took %s and finished %s ago)" + " recovered: " % (abbreviate_time(end-start), + abbreviate_time(time.time() - end)), + self.format_recovered(last["space-recovered"], "actual")) p = T.ul() + def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) saw = self.format_recovered(last["space-recovered"], "examined") add("and saw a total of ", saw) @@ -264,4 +269,37 @@ class StorageStatus(MultiFormatPage): for corrupt_share in last["corrupt-shares"] ] ]]]) - return ctx.tag[p] + return tag(p) + + def format_recovered(self, sr, a): + def maybe(d): + if d is None: + return "?" + return "%d" % d + return "%s shares, %s buckets (%s mutable / %s immutable), %s (%s / %s)" % \ + (maybe(sr["%s-shares" % a]), + maybe(sr["%s-buckets" % a]), + maybe(sr["%s-buckets-mutable" % a]), + maybe(sr["%s-buckets-immutable" % a]), + abbreviate_space(sr["%s-diskbytes" % a]), + abbreviate_space(sr["%s-diskbytes-mutable" % a]), + abbreviate_space(sr["%s-diskbytes-immutable" % a]), + ) + +class StorageStatus(MultiFormatResource): + def __init__(self, storage, nickname=""): + super(StorageStatus, self).__init__() + self.storage = storage + self.nickname = nickname + + def render_HTML(self, req): + return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + + def render_JSON(self, req): + req.setHeader("content-type", "text/plain") + d = {"stats": self.storage.get_stats(), + "bucket-counter": self.storage.bucket_counter.get_state(), + "lease-checker": self.storage.lease_checker.get_state(), + "lease-checker-progress": self.storage.lease_checker.get_progress(), + } + return json.dumps(d, indent=1) + "\n" diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d97daf9af..cfd7a860c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - Storage Server Status @@ -7,19 +7,19 @@ -
+

Storage Server Status

- +
- - + + - - + + - - + + - - + + - - + + - - + +
Total disk space:()()
Disk space used:- ()- ()
@@ -28,18 +28,18 @@
Disk space free (root):()() [see 1]
Disk space free (non-root):()() [see 2]
Reserved space:- ()- ()
@@ -48,23 +48,31 @@
Space Available to Tahoe:()()
    -
  • Server Nickname:
    • -
  • Server Nodeid:
    • -
  • Accepting new shares: -
    • +
  • Server Nickname: + + + +
    • +
  • Server Nodeid: + + + +
    • +
  • Accepting new shares: +
  • Total buckets: - + (the number of files and directories for which this server is holding a share)
      -
    • +
@@ -72,11 +80,11 @@

Lease Expiration Crawler

    -
  • Expiration
    • -
  • -
  • -
  • -
  • +
  • Expiration
    • +
  • +
  • +
  • +
From b29652e0f0d27aafd52771b3f7b57f511fcec076 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 22:59:45 -0500 Subject: [PATCH 002/134] Add `StorageStatus::renderSynchronously` Related to ticket:3247 test_storage.py wants a `StorageStatus::renderSynchronously()` method and a `StorageStatus::renderHTTP()` method. Let us begin with the goofy first-cut. Both these methods are not only wrong, but they will also not please the test suite. However error messages produced in CI can be shared, and that way I can hopefully get unstuck. --- src/allmydata/web/storage.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 3000fb70d..8f36cf44d 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,7 +1,8 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, renderer, Element, renderElement, XMLFile +from twisted.web.template import tags as T, \ + renderer, Element, renderElement, XMLFile from allmydata.web.common import ( abbreviate_time, MultiFormatResource @@ -303,3 +304,14 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self.storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" + + def renderSynchronously(self): + # to appease the test suite. + elem = StorageStatusElement(self.storage, self.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result + + def renderHTTP(self, ctx=None): + # to appease the test suite. + self.renderSynchronously() From d3790a4d4227f31f06a35042448a1a119fa4658e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 4 Feb 2020 23:16:48 -0500 Subject: [PATCH 003/134] Add missing `flattenString` import --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 8f36cf44d..a912d79f6 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -2,7 +2,7 @@ import time, json from twisted.python.filepath import FilePath from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile + renderer, Element, renderElement, XMLFile, flattenString from allmydata.web.common import ( abbreviate_time, MultiFormatResource From c019c7e9556e31c599997ce6463fe679668c3a73 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:05:33 -0500 Subject: [PATCH 004/134] Second version of renderSynchronously --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a912d79f6..4ad5a2baf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -310,7 +310,7 @@ class StorageStatus(MultiFormatResource): elem = StorageStatusElement(self.storage, self.nickname) result = [] flattenString(None, elem).addCallback(result.append) - return result + return result[0] def renderHTTP(self, ctx=None): # to appease the test suite. From 4e81a3a0a2906a2e2c3774a126afa750b7f9bf3c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 22:09:16 -0500 Subject: [PATCH 005/134] Check storage server status before using it --- src/allmydata/web/storage.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 4ad5a2baf..9a99286bf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -31,9 +31,13 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): + if not self.storage: + return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def get_stat(self, key): + if not self.storage: + return None return self.storage.get_stats().get(key) def str(self, tag, val): @@ -101,6 +105,8 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): + if not self.storage: + return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -109,6 +115,8 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): + if not self.storage: + return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -145,6 +153,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -153,6 +163,8 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -176,12 +188,16 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -239,6 +255,8 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): + if not self.storage: + return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From c88c97aad540b928129944bdcd349df6c7df72de Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 5 Feb 2020 23:09:39 -0500 Subject: [PATCH 006/134] Use right syntax for twisted.web.template tags --- src/allmydata/web/storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9a99286bf..1e206d8cf 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -212,7 +212,7 @@ class StorageStatusElement(Element): p = T.ul() def add(*pieces): - p[T.li[pieces]] + p(T.li(pieces)) def maybe(d): if d is None: @@ -248,9 +248,9 @@ class StorageStatusElement(Element): if so_far["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in so_far["corrupt-shares"] ] - ]]]) + )))) return tag("Current cycle:", p) @renderer @@ -284,9 +284,9 @@ class StorageStatusElement(Element): if last["corrupt-shares"]: add("Corrupt shares:", - T.ul[ [T.li[ ["SI %s shnum %d" % corrupt_share + T.ul( (T.li( ["SI %s shnum %d" % corrupt_share for corrupt_share in last["corrupt-shares"] ] - ]]]) + )))) return tag(p) From c7a63f957d4a476092f3bcff70e57dacc9d47c60 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:17:35 -0500 Subject: [PATCH 007/134] Refactor so that test_util pass --- src/allmydata/test/test_storage.py | 5 +++-- src/allmydata/web/storage.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 9f3aee9b8..85f4dd2bd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,7 +30,8 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, remove_prefix +from allmydata.web.storage import StorageStatus, StorageStatusElement, \ + remove_prefix from allmydata.storage_client import ( _StorageServer, ) @@ -4208,7 +4209,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) def test_util(self): - w = StorageStatus(None) + w = StorageStatusElement(None, None) self.failUnlessEqual(w.render_space(None, None), "?") self.failUnlessEqual(w.render_space(None, 10e6), "10000000") self.failUnlessEqual(w.render_abbrev_space(None, None), "?") diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1e206d8cf..6b033bc5a 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,6 +50,16 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) + def render_abbrev_space(self, ctx, size): + if size is None: + return "?" + return abbreviate_space(size) + + def render_space(self, ctx, size): + if size is None: + return "?" + return "%d" % size + @renderer def disk_total(self, req, tag): return self.str(tag, self.get_stat("storage_server.disk_total")) From 0993e610468afe97876c9e4af4367ac9a6a7bde4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:20:45 -0500 Subject: [PATCH 008/134] Drop unused `ctx` argument from render_space methods --- src/allmydata/test/test_storage.py | 8 ++++---- src/allmydata/web/storage.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 85f4dd2bd..6f4c455f2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -4210,9 +4210,9 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_util(self): w = StorageStatusElement(None, None) - self.failUnlessEqual(w.render_space(None, None), "?") - self.failUnlessEqual(w.render_space(None, 10e6), "10000000") - self.failUnlessEqual(w.render_abbrev_space(None, None), "?") - self.failUnlessEqual(w.render_abbrev_space(None, 10e6), "10.00 MB") + self.failUnlessEqual(w.render_space(None), "?") + self.failUnlessEqual(w.render_space(10e6), "10000000") + self.failUnlessEqual(w.render_abbrev_space(None), "?") + self.failUnlessEqual(w.render_abbrev_space(10e6), "10.00 MB") self.failUnlessEqual(remove_prefix("foo.bar", "foo."), "bar") self.failUnlessEqual(remove_prefix("foo.bar", "baz."), None) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 6b033bc5a..88f099b24 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -50,12 +50,12 @@ class StorageStatusElement(Element): return tag("?") return tag(abbreviate_space(val)) - def render_abbrev_space(self, ctx, size): + def render_abbrev_space(self, size): if size is None: return "?" return abbreviate_space(size) - def render_space(self, ctx, size): + def render_space(self, size): if size is None: return "?" return "%d" % size From d46df30bd059633d68dee3eb5d42ba5e375d0d5c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 05:49:11 -0500 Subject: [PATCH 009/134] Use render_space methods to render space --- src/allmydata/web/storage.py | 46 +++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 88f099b24..496766b92 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -40,16 +40,6 @@ class StorageStatusElement(Element): return None return self.storage.get_stats().get(key) - def str(self, tag, val): - if val is None: - return tag("?") - return tag(str(val)) - - def abbr(self, tag, val): - if val is None: - return tag("?") - return tag(abbreviate_space(val)) - def render_abbrev_space(self, size): if size is None: return "?" @@ -62,51 +52,63 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_total")) + val = self.get_stat("storage_server.disk_total") + return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_used")) + val = self.get_stat("storage_server.disk_used") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_root")) + val = self.get_stat("storage_server.disk_free_for_root") + return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_free_for_nonroot")) + val = self.get_stat("storage_server.disk_free_for_nonroot") + return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - return self.str(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.reserved_space")) + val = self.get_stat("storage_server.reserved_space") + return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - return self.str(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - return self.abbr(tag, self.get_stat("storage_server.disk_avail")) + val = self.get_stat("storage_server.disk_avail") + return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): From b14f36082c148119cc9814c56a02a4155447032a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 06:22:25 -0500 Subject: [PATCH 010/134] Use within table cells --- src/allmydata/web/storage_status.xhtml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index cfd7a860c..6e0e26008 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -13,13 +13,13 @@ - - + + - - + + - - + + - - + + - - + + - - + +
Total disk space:()()
Disk space used:- ()- ()
@@ -28,18 +28,18 @@
Disk space free (root):()() [see 1]
Disk space free (non-root):()() [see 2]
Reserved space:- ()- ()
@@ -48,8 +48,8 @@
Space Available to Tahoe:()()
From 2df2ae92d56722ee9d810a678d9b74fe197cb1ab Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 07:34:55 -0500 Subject: [PATCH 011/134] Fix test_storage.WebStatus.test_no_server failure --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 496766b92..c2e517dce 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -161,7 +161,7 @@ class StorageStatusElement(Element): def storage_running(self, req, tag): if self.storage: return tag - return tag("No Storage Server Running") + return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): From b1c78244abeab83e0061e7586b26d4b44911a85d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 08:19:29 -0500 Subject: [PATCH 012/134] Use parentheses in import statement --- src/allmydata/web/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index c2e517dce..d45541457 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -1,8 +1,14 @@ import time, json from twisted.python.filepath import FilePath -from twisted.web.template import tags as T, \ - renderer, Element, renderElement, XMLFile, flattenString +from twisted.web.template import ( + Element, + XMLFile, + tags as T, + renderer, + renderElement, + flattenString +) from allmydata.web.common import ( abbreviate_time, MultiFormatResource From d3ff578640f85181fffd787b5209e7847033d8e9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 09:55:33 -0500 Subject: [PATCH 013/134] Use parentheses in test suite's import statement --- src/allmydata/test/test_storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 6f4c455f2..6956fc757 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,8 +30,11 @@ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin from allmydata.test.no_network import NoNetworkServer -from allmydata.web.storage import StorageStatus, StorageStatusElement, \ +from allmydata.web.storage import ( + StorageStatus, + StorageStatusElement, remove_prefix +) from allmydata.storage_client import ( _StorageServer, ) From 227d06fe64533ba45b1319b7f498e0df3492f670 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 11 Feb 2020 17:02:14 -0500 Subject: [PATCH 014/134] Add docstrings to StorageStatusElement --- src/allmydata/web/storage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d45541457..df8354a38 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -24,9 +24,15 @@ def remove_prefix(s, prefix): class StorageStatusElement(Element): + """Class to render a storage status page.""" + loader = XMLFile(FilePath(__file__).sibling("storage_status.xhtml")) - def __init__(self, storage, nickname): + def __init__(self, storage, nickname=""): + """ + :param _StorageServer storage: data about storage. + :param string nickname: friendly name for storage. + """ super(StorageStatusElement, self).__init__() self.storage = storage self.nick = nickname From f22417e51b4fadbba09cf2ad882605c5aecaef20 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:14:24 -0500 Subject: [PATCH 015/134] Rename function for clarity --- src/allmydata/web/storage.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index df8354a38..a5d65fa85 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -47,7 +47,7 @@ class StorageStatusElement(Element): return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) - def get_stat(self, key): + def _get_storage_stat(self, key): if not self.storage: return None return self.storage.get_stats().get(key) @@ -64,67 +64,67 @@ class StorageStatusElement(Element): @renderer def disk_total(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_space(val)) @renderer def disk_total_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_total") + val = self._get_storage_stat("storage_server.disk_total") return tag(self.render_abbrev_space(val)) @renderer def disk_used(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_space(val)) @renderer def disk_used_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_used") + val = self._get_storage_stat("storage_server.disk_used") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_root(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_space(val)) @renderer def disk_free_for_root_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_root") + val = self._get_storage_stat("storage_server.disk_free_for_root") return tag(self.render_abbrev_space(val)) @renderer def disk_free_for_nonroot(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_space(val)) @renderer def disk_free_for_nonroot_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_free_for_nonroot") + val = self._get_storage_stat("storage_server.disk_free_for_nonroot") return tag(self.render_abbrev_space(val)) @renderer def reserved_space(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_space(val)) @renderer def reserved_space_abbrev(self, req, tag): - val = self.get_stat("storage_server.reserved_space") + val = self._get_storage_stat("storage_server.reserved_space") return tag(self.render_abbrev_space(val)) @renderer def disk_avail(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_space(val)) @renderer def disk_avail_abbrev(self, req, tag): - val = self.get_stat("storage_server.disk_avail") + val = self._get_storage_stat("storage_server.disk_avail") return tag(self.render_abbrev_space(val)) @renderer def accepting_immutable_shares(self, req, tag): - accepting = self.get_stat("storage_server.accepting_immutable_shares") + accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") return {True: "Yes", False: "No"}[bool(accepting)] @renderer From 6e9a4e30d733c2a08d801df03f1107d9177ec76e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 12 Feb 2020 12:15:26 -0500 Subject: [PATCH 016/134] Add a docstring --- src/allmydata/web/storage.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index a5d65fa85..263fa9344 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -48,6 +48,30 @@ class StorageStatusElement(Element): return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): + """Get storage server statistics. + + Storage Server keeps a dict that contains various usage and + latency statistics. The dict looks like this: + + { + 'storage_server.accepting_immutable_shares': 1, + 'storage_server.allocated': 0, + 'storage_server.disk_avail': 106539192320, + 'storage_server.disk_free_for_nonroot': 106539192320, + 'storage_server.disk_free_for_root': 154415284224, + 'storage_server.disk_total': 941088460800, + 'storage_server.disk_used': 786673176576, + 'storage_server.latencies.add-lease.01_0_percentile': None, + 'storage_server.latencies.add-lease.10_0_percentile': None, + ... + } + + ``StorageServer.get_stats()`` returns the above dict. Storage + status page uses a subset of the items in the dict, concerning + disk usage. + + :param str key: storage server statistic we want to know. + """ if not self.storage: return None return self.storage.get_stats().get(key) From 9b51bdf7faca3314e5dff2de22b8e035b9114fec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 12:25:20 -0500 Subject: [PATCH 017/134] Return result from renderHTTP Bogus renderHTTP, but test failures are now down to three. We just need to handle requests for JSON now. --- src/allmydata/web/storage.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 263fa9344..1656df70b 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -378,6 +378,18 @@ class StorageStatus(MultiFormatResource): flattenString(None, elem).addCallback(result.append) return result[0] + # to appease the test suite def renderHTTP(self, ctx=None): - # to appease the test suite. - self.renderSynchronously() + """Send HTML or JSON formatted data, based on request. + + This function contains a bit of nevow-ism, but since this is + only called from the test suite, the nevow-ism should go away + as we update things. + + :param _nevow.context.WovenContext ctx: context is passed on + from the test suite. We get a request out of this + context, and use the request to render a result. + + """ + from nevow.inevow import IRequest + return self.render(IRequest(ctx)) From 7a053ddeff85d32376c12582ca35a825e708704a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 15:29:42 -0500 Subject: [PATCH 018/134] Use explicit `None`-check on self.storage --- src/allmydata/web/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 1656df70b..76a09fda3 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -43,7 +43,7 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if not self.storage: + if self.storage is None: return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) From e2fc1fc07ea56cd6b4805864d1d275170e0c261e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 16:49:15 -0500 Subject: [PATCH 019/134] Move `renderSynchronously` to test suite --- src/allmydata/test/test_storage.py | 36 +++++++++++++++++++----------- src/allmydata/web/storage.py | 10 +-------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 6956fc757..f37401cc2 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -4,6 +4,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service +from twisted.web.template import flattenString from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -2963,6 +2964,15 @@ def remove_tags(s): s = re.sub(r'\s+', ' ', s) return s +def renderSynchronously(ss): + """ + :param _StorageStatus ss: a StorageStatus instance. + """ + elem = StorageStatusElement(ss.storage, ss.nickname) + result = [] + flattenString(None, elem).addCallback(result.append) + return result[0] + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -2999,7 +3009,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): w = StorageStatus(ss) # this sample is before the crawler has started doing anything - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -3022,7 +3032,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(state["last-complete-prefix"], ss.bucket_counter.prefixes[0]) ss.bucket_counter.cpu_slice = 100.0 # finish as fast as possible - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn(" Current crawl ", s) self.failUnlessIn(" (next work in ", s) @@ -3034,7 +3044,7 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): d.addCallback(lambda ignored: self.poll(_watch)) def _check2(ignored): ss.bucket_counter.cpu_slice = orig_cpu_slice - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("Total buckets: 0 (the number of", s) self.failUnless("Next crawl in 59 minutes" in s or "Next crawl in 60 minutes" in s, s) @@ -3096,20 +3106,20 @@ class BucketCounter(unittest.TestCase, pollmixin.PollMixin): def _check_1(ignored): # no ETA is available yet - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (next work", s) def _check_2(ignored): # one prefix has finished, so an ETA based upon that elapsed time # should be available. - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) def _check_3(ignored): # two prefixes have finished - html = w.renderSynchronously() + html = renderSynchronously(w) s = remove_tags(html) self.failUnlessIn("complete (ETA ", s) d.callback("done") @@ -4064,7 +4074,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): def test_no_server(self): w = StorageStatus(None) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

No Storage Server Running

", html) def test_status(self): @@ -4110,7 +4120,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: Yes", s) @@ -4130,7 +4140,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4166,7 +4176,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) @@ -4184,7 +4194,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, readonly_storage=True) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Accepting new shares: No", s) @@ -4195,7 +4205,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) @@ -4206,7 +4216,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, "\x00" * 20, reserved_space=10e6) ss.setServiceParent(self.s) w = StorageStatus(ss) - html = w.renderSynchronously() + html = renderSynchronously(w) self.failUnlessIn("

Storage Server Status

", html) s = remove_tags(html) self.failUnlessIn("Reserved space: - 10.00 MB (10000000)", s) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 76a09fda3..e9c2f6bb5 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -6,8 +6,7 @@ from twisted.web.template import ( XMLFile, tags as T, renderer, - renderElement, - flattenString + renderElement ) from allmydata.web.common import ( abbreviate_time, @@ -371,13 +370,6 @@ class StorageStatus(MultiFormatResource): } return json.dumps(d, indent=1) + "\n" - def renderSynchronously(self): - # to appease the test suite. - elem = StorageStatusElement(self.storage, self.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] - # to appease the test suite def renderHTTP(self, ctx=None): """Send HTML or JSON formatted data, based on request. From c061f6830e66ff7770bc0af43eb0442fbdf0f441 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:00:54 -0500 Subject: [PATCH 020/134] Use `successResultOf` in `renderSynchronously` Get rid of [].append trick when dealing with the deferred. --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f37401cc2..0a176bd60 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2969,9 +2969,8 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss.storage, ss.nickname) - result = [] - flattenString(None, elem).addCallback(result.append) - return result[0] + deferred = flattenString(None, elem) + return unittest.TestCase().successResultOf(deferred) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): From aab940f65fb95815c9998b79876adeda5a0fb5f8 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:18:37 -0500 Subject: [PATCH 021/134] Remove redundant `None`-checks on `self.storage` When no storage is up, `storage_running()` renderer will return a big honking `no storage server running` message, and no further renderers will be invoked. Therefore the extra defense is probably not required. (I tested this hypothesis. The extra defense is not required, unless there's something I have not seen.) --- src/allmydata/web/storage.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index e9c2f6bb5..cbee2780f 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -42,8 +42,6 @@ class StorageStatusElement(Element): @renderer def nodeid(self, req, tag): - if self.storage is None: - return tag("No storage server running.") return idlib.nodeid_b2a(self.storage.my_nodeid) def _get_storage_stat(self, key): @@ -71,8 +69,6 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - if not self.storage: - return None return self.storage.get_stats().get(key) def render_abbrev_space(self, size): @@ -152,8 +148,6 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - if not self.storage: - return tag("No storage server running.") s = self.storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: @@ -162,8 +156,6 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - if not self.storage: - return tag("No storage server running.") p = self.storage.bucket_counter.get_progress() return self.format_crawler_progress(p) @@ -200,8 +192,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_enabled(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") @@ -210,8 +200,6 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: @@ -235,16 +223,12 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: @@ -302,8 +286,6 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - if not self.storage: - return tag("No storage server running.") lc = self.storage.lease_checker h = lc.get_state()["history"] if not h: From 0382b1ec63cfdcebf5f76f03ee939edbd0901d35 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 17:23:35 -0500 Subject: [PATCH 022/134] Use Unicode strings to render space --- src/allmydata/web/storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index cbee2780f..607eeff79 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -73,13 +73,13 @@ class StorageStatusElement(Element): def render_abbrev_space(self, size): if size is None: - return "?" + return u"?" return abbreviate_space(size) def render_space(self, size): if size is None: - return "?" - return "%d" % size + return u"?" + return u"%d" % size @renderer def disk_total(self, req, tag): From 36a486426e970d86fb2dea42ab9ffba0dfc6238c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 13 Feb 2020 22:02:30 -0500 Subject: [PATCH 023/134] Mark `format_recovered` as static method --- src/allmydata/web/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 607eeff79..d670c4231 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -319,7 +319,8 @@ class StorageStatusElement(Element): return tag(p) - def format_recovered(self, sr, a): + @staticmethod + def format_recovered(sr, a): def maybe(d): if d is None: return "?" From 9bb7812148b7aa17cfa641f080ab8e3eb2bce642 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 07:25:49 -0500 Subject: [PATCH 024/134] Add news fragment --- newsfragments/3247.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3247.minor diff --git a/newsfragments/3247.minor b/newsfragments/3247.minor new file mode 100644 index 000000000..e69de29bb From 7625d959bc2735816710af01e80073b102312e17 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 08:26:44 -0500 Subject: [PATCH 025/134] Use to render node nickname and id CI did not like the old way, but it passed in my system. Odd. --- src/allmydata/web/storage_status.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 6e0e26008..8f74c478c 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,12 +57,12 @@
  • Server Nickname: - +
  • Server Nodeid: - +
  • Accepting new shares: From ff019e5b12d3210046e269519a1e474f1d99ceb3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 10:10:48 -0500 Subject: [PATCH 026/134] Use BeautifulSoup to check favicon in storage page --- src/allmydata/test/web/test_web.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index f84923521..19753d3b7 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -963,8 +963,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def test_storage(self): d = self.GET("/storage") def _check(res): - self.failUnlessIn('Storage Server Status', res) - self.failUnlessIn(FAVICON_MARKUP, res) + soup = BeautifulSoup(res, 'html5lib') + assert_soup_has_text(self, soup, 'Storage Server Status') + assert_soup_has_favicon(self, soup) res_u = res.decode('utf-8') self.failUnlessIn(u'
  • Server Nickname: fake_nickname \u263A
    • ', res_u) d.addCallback(_check) From 0cbe2871fd80ffd186d54848995c6b6c91ee5ca1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 14 Feb 2020 11:42:42 -0500 Subject: [PATCH 027/134] Give the suite the precise string it wants --- src/allmydata/web/storage_status.xhtml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index 8f74c478c..d052fbabd 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -55,16 +55,8 @@
      -
    • Server Nickname: - - - -
      • -
    • Server Nodeid: - - - -
      • +
    • Server Nickname:
      • +
    • Server Nodeid:
    • Accepting new shares:
    • Total buckets: From 6c3256517a0fa8cd1b8c00267d0f80325f868a70 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 17 Feb 2020 12:52:48 -0500 Subject: [PATCH 028/134] Use slots to render storage stats table --- src/allmydata/web/storage.py | 80 ++++++++------------------ src/allmydata/web/storage_status.xhtml | 26 ++++----- 2 files changed, 36 insertions(+), 70 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d670c4231..d26f4a5dd 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -82,64 +82,30 @@ class StorageStatusElement(Element): return u"%d" % size @renderer - def disk_total(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_space(val)) + def storage_stats(self, req, tag): + # Render storage status table that appears near the top of the page. + total = self._get_storage_stat("storage_server.disk_total") + used = self._get_storage_stat("storage_server.disk_used") + free_root = self._get_storage_stat("storage_server.disk_free_for_root") + free_nonroot = self._get_storage_stat("storage_server.disk_free_for_nonroot") + reserved = self._get_storage_stat("storage_server.reserved_space") + available = self._get_storage_stat("storage_server.disk_avail") - @renderer - def disk_total_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_total") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_used(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_space(val)) - - @renderer - def disk_used_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_used") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_root(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_root_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_root") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_free_for_nonroot(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_space(val)) - - @renderer - def disk_free_for_nonroot_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_free_for_nonroot") - return tag(self.render_abbrev_space(val)) - - @renderer - def reserved_space(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_space(val)) - - @renderer - def reserved_space_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.reserved_space") - return tag(self.render_abbrev_space(val)) - - @renderer - def disk_avail(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_space(val)) - - @renderer - def disk_avail_abbrev(self, req, tag): - val = self._get_storage_stat("storage_server.disk_avail") - return tag(self.render_abbrev_space(val)) + tag.fillSlots( + disk_total = self.render_space(total), + disk_total_abbrev = self.render_abbrev_space(total), + disk_used = self.render_space(used), + disk_used_abbrev = self.render_abbrev_space(used), + disk_free_for_root = self.render_space(free_root), + disk_free_for_root_abbrev = self.render_abbrev_space(free_root), + disk_free_for_nonroot = self.render_space(free_nonroot), + disk_free_for_nonroot_abbrev = self.render_abbrev_space(free_nonroot), + reserved_space = self.render_space(reserved), + reserved_space_abbrev = self.render_abbrev_space(reserved), + disk_avail = self.render_space(available), + disk_avail_abbrev = self.render_abbrev_space(available) + ) + return tag @renderer def accepting_immutable_shares(self, req, tag): diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index d052fbabd..bbf3d2c8a 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -11,15 +11,15 @@

      Storage Server Status

      - +
      - - + + - - + + - - + + - - + + - - + + - - + +
      Total disk space:()()
      Disk space used:- ()- ()
      @@ -28,18 +28,18 @@
      Disk space free (root):()() [see 1]
      Disk space free (non-root):()() [see 2]
      Reserved space:- ()- ()
      @@ -48,8 +48,8 @@
      Space Available to Tahoe:()()
      From 554c477cea5a3c777794923d406137aa1be46f18 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 25 Feb 2020 08:52:45 -0500 Subject: [PATCH 029/134] Prefix member variable with "_" --- src/allmydata/test/test_storage.py | 2 +- src/allmydata/web/storage.py | 40 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 0a176bd60..56d281688 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2968,7 +2968,7 @@ def renderSynchronously(ss): """ :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss.storage, ss.nickname) + elem = StorageStatusElement(ss._storage, ss._nickname) deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index d26f4a5dd..9b3daa0cb 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -33,16 +33,16 @@ class StorageStatusElement(Element): :param string nickname: friendly name for storage. """ super(StorageStatusElement, self).__init__() - self.storage = storage - self.nick = nickname + self._storage = storage + self._nickname = nickname @renderer def nickname(self, req, tag): - return self.nick + return self._nickname @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self.storage.my_nodeid) + return idlib.nodeid_b2a(self._storage.my_nodeid) def _get_storage_stat(self, key): """Get storage server statistics. @@ -69,7 +69,7 @@ class StorageStatusElement(Element): :param str key: storage server statistic we want to know. """ - return self.storage.get_stats().get(key) + return self._storage.get_stats().get(key) def render_abbrev_space(self, size): if size is None: @@ -114,7 +114,7 @@ class StorageStatusElement(Element): @renderer def last_complete_bucket_count(self, req, tag): - s = self.storage.bucket_counter.get_state() + s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: return "Not computed yet" @@ -122,7 +122,7 @@ class StorageStatusElement(Element): @renderer def count_crawler_status(self, req, tag): - p = self.storage.bucket_counter.get_progress() + p = self._storage.bucket_counter.get_progress() return self.format_crawler_progress(p) def format_crawler_progress(self, p): @@ -152,13 +152,13 @@ class StorageStatusElement(Element): @renderer def storage_running(self, req, tag): - if self.storage: + if self._storage: return tag return T.h1("No Storage Server Running") @renderer def lease_expiration_enabled(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.expiration_enabled: return tag("Enabled: expired leases will be removed") else: @@ -166,7 +166,7 @@ class StorageStatusElement(Element): @renderer def lease_expiration_mode(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker if lc.mode == "age": if lc.override_lease_duration is None: tag("Leases will expire naturally, probably 31 days after " @@ -189,13 +189,13 @@ class StorageStatusElement(Element): @renderer def lease_current_cycle_progress(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() return tag(self.format_crawler_progress(p)) @renderer def lease_current_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker p = lc.get_progress() if not p["cycle-in-progress"]: return "" @@ -252,7 +252,7 @@ class StorageStatusElement(Element): @renderer def lease_last_cycle_results(self, req, tag): - lc = self.storage.lease_checker + lc = self._storage.lease_checker h = lc.get_state()["history"] if not h: return "" @@ -304,18 +304,18 @@ class StorageStatusElement(Element): class StorageStatus(MultiFormatResource): def __init__(self, storage, nickname=""): super(StorageStatus, self).__init__() - self.storage = storage - self.nickname = nickname + self._storage = storage + self._nickname = nickname def render_HTML(self, req): - return renderElement(req, StorageStatusElement(self.storage, self.nickname)) + return renderElement(req, StorageStatusElement(self._storage, self._nickname)) def render_JSON(self, req): req.setHeader("content-type", "text/plain") - d = {"stats": self.storage.get_stats(), - "bucket-counter": self.storage.bucket_counter.get_state(), - "lease-checker": self.storage.lease_checker.get_state(), - "lease-checker-progress": self.storage.lease_checker.get_progress(), + d = {"stats": self._storage.get_stats(), + "bucket-counter": self._storage.bucket_counter.get_state(), + "lease-checker": self._storage.lease_checker.get_state(), + "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" From 110734daf065e7b2706e7d92b2361c21954c11eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:45:18 -0500 Subject: [PATCH 030/134] Use a helper to exercise render() in storage test cases --- src/allmydata/test/test_storage.py | 50 ++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 56d281688..86b23cd48 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -30,6 +30,7 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin from allmydata.test.common_web import WebRenderingMixin +from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2972,6 +2973,29 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) +def renderDeferred(resource, **kwargs): + """ + Use this to exercise an overridden MultiFormatResource.render(), + usually for output=json or render_GET. It returns a Deferred. + + :param _MultiFormatResource resource: an HTTP resource to be rendered. + + """ + # We should be using twisted.web's DummyRequest here instead of + # nevow's FakeRequest, but right now it is a bit of a problem: see + # web/common.py. MultiFormatResource.render() makes a get_arg() + # call, which does a IRequest(ctx_or_req). IRequest can handle + # FakeRequest, but it can't handle DummyRequest. + req = FakeRequest(**kwargs) + req.fields = None + d = defer.maybeDeferred(resource.render, req) + def _done(res): + if isinstance(res, str): + return res + req.v + return req.v + d.addCallback(_done) + return d + class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): BucketCountingCrawler.finished_prefix(self, cycle, prefix) @@ -3291,7 +3315,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failIfEqual(sr2["configured-diskbytes"], None) self.failIfEqual(sr2["original-sharebytes"], None) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) self.failUnlessIn("So far, this cycle has examined " @@ -3366,7 +3390,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_leases(mutable_si_2), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("recovered: 0 shares, 0 buckets " @@ -3466,7 +3490,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3525,7 +3549,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled: expired leases will be removed", s) @@ -3610,7 +3634,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d2.addCallback(_after_first_bucket) return d2 d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html_in_cycle(html): s = remove_tags(html) # the first bucket encountered gets deleted, and its prefix @@ -3671,7 +3695,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnless(rec["configured-diskbytes"] >= 0, rec["configured-diskbytes"]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Expiration Enabled:" @@ -3733,7 +3757,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_3), 1) self.failUnlessEqual(count_leases(mutable_si_3), 2) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: immutable.", s) @@ -3790,7 +3814,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): self.failUnlessEqual(count_shares(mutable_si_2), 0) self.failUnlessEqual(count_shares(mutable_si_3), 0) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render1(webstatus)) + d.addCallback(lambda ign: renderDeferred(webstatus)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("The following sharetypes will be expired: mutable.", s) @@ -4021,7 +4045,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): # it also turns all tuples into lists self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4046,7 +4070,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): corrupt_shares = last["corrupt-shares"] self.failUnlessEqual(corrupt_shares, [[first_b32, 0]]) d.addCallback(_check_json_history) - d.addCallback(lambda ign: self.render1(w)) + d.addCallback(lambda ign: renderDeferred(w)) def _check_html_history(html): s = remove_tags(html) self.failUnlessIn("Corrupt shares: SI %s shnum 0" % first_b32, s) @@ -4060,7 +4084,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): @@ -4083,7 +4107,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): ss = StorageServer(basedir, nodeid) ss.setServiceParent(self.s) w = StorageStatus(ss, "nickname") - d = self.render1(w) + d = renderDeferred(w) def _check_html(html): self.failUnlessIn("

      Storage Server Status

      ", html) s = remove_tags(html) @@ -4104,7 +4128,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): return d def render_json(self, page): - d = self.render1(page, args={"t": ["json"]}) + d = renderDeferred(page, args={"t": ["json"]}) return d def test_status_no_disk_stats(self): From f1fe3a75884c2073310727a5f4c4127bfb928645 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:46:09 -0500 Subject: [PATCH 031/134] Get rid of WebRenderingMixin in storage test --- src/allmydata/test/test_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 86b23cd48..cff8c3e22 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -29,7 +29,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from allmydata.test.common_web import WebRenderingMixin from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( @@ -3185,7 +3184,7 @@ class InstrumentedStorageServer(StorageServer): class No_ST_BLOCKS_StorageServer(StorageServer): LeaseCheckerClass = No_ST_BLOCKS_LeaseCheckingCrawler -class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() @@ -4087,7 +4086,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): d = renderDeferred(page, args={"t": ["json"]}) return d -class WebStatus(unittest.TestCase, pollmixin.PollMixin, WebRenderingMixin): +class WebStatus(unittest.TestCase, pollmixin.PollMixin): def setUp(self): self.s = service.MultiService() From bae32179bf467e8802d235e5bcd4c1c18411a5c9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 5 Mar 2020 15:57:12 -0500 Subject: [PATCH 032/134] Remove StorageStatus.renderHTTP This was added to please the test suite. Pleased to remove it! --- src/allmydata/web/storage.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 9b3daa0cb..63ac47282 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -318,19 +318,3 @@ class StorageStatus(MultiFormatResource): "lease-checker-progress": self._storage.lease_checker.get_progress(), } return json.dumps(d, indent=1) + "\n" - - # to appease the test suite - def renderHTTP(self, ctx=None): - """Send HTML or JSON formatted data, based on request. - - This function contains a bit of nevow-ism, but since this is - only called from the test suite, the nevow-ism should go away - as we update things. - - :param _nevow.context.WovenContext ctx: context is passed on - from the test suite. We get a request out of this - context, and use the request to render a result. - - """ - from nevow.inevow import IRequest - return self.render(IRequest(ctx)) From 9c7357bc61dd74e4b15ae0da2d28a5275acd6b5e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 30 Mar 2020 16:15:44 -0400 Subject: [PATCH 033/134] Remove an extraneous directive `t:data` is not really a Twisted template directive. Added my mistake, removing now. --- src/allmydata/web/storage_status.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/storage_status.xhtml b/src/allmydata/web/storage_status.xhtml index bbf3d2c8a..354f9f177 100644 --- a/src/allmydata/web/storage_status.xhtml +++ b/src/allmydata/web/storage_status.xhtml @@ -57,7 +57,7 @@
      • Server Nickname:
      • Server Nodeid:
        • -
      • Accepting new shares: +
      • Accepting new shares:
      • Total buckets: From 8c92187d9263fa8d092c62c1305200191e850ec5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:00:55 -0400 Subject: [PATCH 034/134] Avoid using nevow FakeRequest in storage test. Use twisted.web.server.Request instead, with a DummyChannel. There's still one line of inevitable nevow now, because of code in web/common.py; but that should be easily replaceable once we switch that over. --- src/allmydata/test/test_storage.py | 47 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index cff8c3e22..40b9b52e9 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -5,6 +5,16 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.application import service from twisted.web.template import flattenString + +# We need to use `nevow.inevow.IRequest` for now for compatibility +# with the code in web/common.py. Once nevow bits are gone from +# web/common.py, we can use `twisted.web.iweb.IRequest` here. +from nevow.inevow import IRequest + +from twisted.web.server import Request +from twisted.web.test.test_web import DummyChannel +from zope.interface import implements + from foolscap.api import fireEventually import itertools from allmydata import interfaces @@ -29,7 +39,6 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SHARE_HASH_CHAIN_SIZE from allmydata.interfaces import BadWriteEnablerError from allmydata.test.common import LoggingServiceParent, ShouldFailMixin -from nevow.testutil import FakeRequest from allmydata.test.no_network import NoNetworkServer from allmydata.web.storage import ( StorageStatus, @@ -2972,28 +2981,20 @@ def renderSynchronously(ss): deferred = flattenString(None, elem) return unittest.TestCase().successResultOf(deferred) -def renderDeferred(resource, **kwargs): - """ - Use this to exercise an overridden MultiFormatResource.render(), - usually for output=json or render_GET. It returns a Deferred. +def renderDeferred(ss): + elem = StorageStatusElement(ss._storage, ss._nickname) + return flattenString(None, elem) - :param _MultiFormatResource resource: an HTTP resource to be rendered. +class JSONRequest(Request): + implements(IRequest) - """ - # We should be using twisted.web's DummyRequest here instead of - # nevow's FakeRequest, but right now it is a bit of a problem: see - # web/common.py. MultiFormatResource.render() makes a get_arg() - # call, which does a IRequest(ctx_or_req). IRequest can handle - # FakeRequest, but it can't handle DummyRequest. - req = FakeRequest(**kwargs) - req.fields = None - d = defer.maybeDeferred(resource.render, req) - def _done(res): - if isinstance(res, str): - return res + req.v - return req.v - d.addCallback(_done) - return d + def __init__(self, **kwargs): + Request.__init__(self, DummyChannel(), **kwargs) + self.args = {"t": ["json"]} + self.fields = {} + +def renderJSON(resource): + return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): def finished_prefix(self, cycle, prefix): @@ -4083,7 +4084,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4127,7 +4128,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): return d def render_json(self, page): - d = renderDeferred(page, args={"t": ["json"]}) + d = renderJSON(page) return d def test_status_no_disk_stats(self): From 3e7dea7dda3bee3dfb2713297c10f3fac1b2698f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:01:22 -0400 Subject: [PATCH 035/134] Wrap renderer results in tags --- src/allmydata/web/storage.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/web/storage.py b/src/allmydata/web/storage.py index 63ac47282..ba6609456 100644 --- a/src/allmydata/web/storage.py +++ b/src/allmydata/web/storage.py @@ -38,11 +38,11 @@ class StorageStatusElement(Element): @renderer def nickname(self, req, tag): - return self._nickname + return tag(self._nickname) @renderer def nodeid(self, req, tag): - return idlib.nodeid_b2a(self._storage.my_nodeid) + return tag(idlib.nodeid_b2a(self._storage.my_nodeid)) def _get_storage_stat(self, key): """Get storage server statistics. @@ -110,20 +110,20 @@ class StorageStatusElement(Element): @renderer def accepting_immutable_shares(self, req, tag): accepting = self._get_storage_stat("storage_server.accepting_immutable_shares") - return {True: "Yes", False: "No"}[bool(accepting)] + return tag({True: "Yes", False: "No"}[bool(accepting)]) @renderer def last_complete_bucket_count(self, req, tag): s = self._storage.bucket_counter.get_state() count = s.get("last-complete-bucket-count") if count is None: - return "Not computed yet" - return str(count) + return tag("Not computed yet") + return tag(str(count)) @renderer def count_crawler_status(self, req, tag): p = self._storage.bucket_counter.get_progress() - return self.format_crawler_progress(p) + return tag(self.format_crawler_progress(p)) def format_crawler_progress(self, p): cycletime = p["estimated-time-per-cycle"] From 8b7ef33b3d3b7caf6a91f6f803111288d2bb5ed4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:05:17 -0400 Subject: [PATCH 036/134] Remove redundant render_json() method --- src/allmydata/test/test_storage.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 40b9b52e9..e20a89da6 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -3399,7 +3399,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): "(2 mutable / 2 immutable),", s) self.failUnlessIn("but expiration was not enabled", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(webstatus)) + d.addCallback(lambda ign: renderJSON(webstatus)) def _check_json(raw): data = json.loads(raw) self.failUnlessIn("lease-checker", data) @@ -4036,7 +4036,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(so_far["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_bucket) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) # grr. json turns all dict keys into strings. @@ -4063,7 +4063,7 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): self.failUnlessEqual(rec["examined-shares"], 3) self.failUnlessEqual(last["corrupt-shares"], [(first_b32, 0)]) d.addCallback(_after_first_cycle) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json_history(raw): data = json.loads(raw) last = data["lease-checker"]["history"]["0"] @@ -4083,9 +4083,6 @@ class LeaseCrawler(unittest.TestCase, pollmixin.PollMixin): d.addBoth(_cleanup) return d - def render_json(self, page): - d = renderJSON(page) - return d class WebStatus(unittest.TestCase, pollmixin.PollMixin): @@ -4116,7 +4113,7 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): self.failUnlessIn("Accepting new shares: Yes", s) self.failUnlessIn("Reserved space: - 0 B (0)", s) d.addCallback(_check_html) - d.addCallback(lambda ign: self.render_json(w)) + d.addCallback(lambda ign: renderJSON(w)) def _check_json(raw): data = json.loads(raw) s = data["stats"] @@ -4127,9 +4124,6 @@ class WebStatus(unittest.TestCase, pollmixin.PollMixin): d.addCallback(_check_json) return d - def render_json(self, page): - d = renderJSON(page) - return d def test_status_no_disk_stats(self): def call_get_disk_stats(whichdir, reserved_space=0): From 72b8f720802f91d5aef25643b326d1ba98b54cc2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:13:08 -0400 Subject: [PATCH 037/134] Add docstrings to storage test helpers --- src/allmydata/test/test_storage.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index e20a89da6..b69713df5 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2975,6 +2975,8 @@ def remove_tags(s): def renderSynchronously(ss): """ + Return fully rendered HTML document. + :param _StorageStatus ss: a StorageStatus instance. """ elem = StorageStatusElement(ss._storage, ss._nickname) @@ -2982,10 +2984,20 @@ def renderSynchronously(ss): return unittest.TestCase().successResultOf(deferred) def renderDeferred(ss): + """ + Return a `Deferred` HTML renderer. + + :param _StorageStatus ss: a StorageStatus instance. + """ elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) class JSONRequest(Request): + """ + A Request with t=json argument added to it. + + This is useful to invoke a Resouce.render_JSON() method. + """ implements(IRequest) def __init__(self, **kwargs): @@ -2994,6 +3006,11 @@ class JSONRequest(Request): self.fields = {} def renderJSON(resource): + """Exercise resouce.render_JSON() + + :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` + that contains a render_JSON() method. + """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From b2b706198042fa39ddc4959d3804647340c8bb24 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 19:18:15 -0400 Subject: [PATCH 038/134] Refactor storage test helpers Rewrite `renderSynchronously()` to use `renderDeferred()` --- src/allmydata/test/test_storage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index b69713df5..f50c3b352 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -2979,9 +2979,7 @@ def renderSynchronously(ss): :param _StorageStatus ss: a StorageStatus instance. """ - elem = StorageStatusElement(ss._storage, ss._nickname) - deferred = flattenString(None, elem) - return unittest.TestCase().successResultOf(deferred) + return unittest.TestCase().successResultOf(renderDeferred(ss)) def renderDeferred(ss): """ From 82cd5a87fe9582dd1de8e021b8b8a8459d3ba499 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 6 Apr 2020 21:48:49 -0400 Subject: [PATCH 039/134] Use DummyRequest in storage tests Using twisted.web.server.Request causes test_new_style_classes to fail like so: Traceback (most recent call last): Failure: testtools.testresult.real._StringException: Traceback (most recent call last): File ".tox/coverage/lib/python2.7/site-packages/allmydata/test/test_python2_regressions.py", line 69, in test_new_style_classes "Expected to find no classic classes.", File ".tox/coverage/lib/python2.7/site-packages/testtools/testcase.py", line 502, in assertThat raise mismatch_error testtools.matchers._impl.MismatchError: !=: reference = set([]) actual = set([]) : Expected to find no classic classes. Seems that `DummyRequest` is an acceptable new style class. --- src/allmydata/test/test_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index f50c3b352..a0816a6ce 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,8 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.server import Request -from twisted.web.test.test_web import DummyChannel +# from twisted.web.server import Request +from twisted.web.test.test_web import DummyRequest from zope.interface import implements from foolscap.api import fireEventually @@ -2990,7 +2990,7 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -class JSONRequest(Request): +class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. @@ -2999,7 +2999,7 @@ class JSONRequest(Request): implements(IRequest) def __init__(self, **kwargs): - Request.__init__(self, DummyChannel(), **kwargs) + DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} self.fields = {} From 201c08dbe5d0e7c585329261e24082670d999f88 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 20 Apr 2020 14:35:32 -0400 Subject: [PATCH 040/134] Declare JSON request interface using @implementer "zope.interface.implements(IRequest)" is deprectated in favor of "@zope.interface.implementer(IRequest)" decorator. --- src/allmydata/test/test_storage.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index a0816a6ce..23920a27a 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,9 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -# from twisted.web.server import Request -from twisted.web.test.test_web import DummyRequest -from zope.interface import implements +from twisted.web.test.requesthelper import DummyRequest +from zope.interface import implementer from foolscap.api import fireEventually import itertools @@ -2990,14 +2989,13 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) +@implementer(IRequest) class JSONRequest(DummyRequest): """ A Request with t=json argument added to it. This is useful to invoke a Resouce.render_JSON() method. """ - implements(IRequest) - def __init__(self, **kwargs): DummyRequest.__init__(self, b"/", **kwargs) self.args = {"t": ["json"]} From b3feaae644de727bf564a50df83e1ff4b733f76a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 21 Apr 2020 10:29:18 -0400 Subject: [PATCH 041/134] Use an inner JSONRequest class with renderJSON Once nevow is removed from web/common.py, we can simplify renderJSON(), like so: def renderJSON(): req = Request() req.args = {"t": ["json"]} req.fields = {} return resource.render(req) But for now we have to live with an inner class that implements the nevow.inevow.IRequest interface. --- src/allmydata/test/test_storage.py | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 23920a27a..6c5f6e937 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -11,7 +11,8 @@ from twisted.web.template import flattenString # web/common.py, we can use `twisted.web.iweb.IRequest` here. from nevow.inevow import IRequest -from twisted.web.test.requesthelper import DummyRequest +from twisted.web.server import Request +from twisted.web.test.requesthelper import DummyChannel from zope.interface import implementer from foolscap.api import fireEventually @@ -2989,24 +2990,20 @@ def renderDeferred(ss): elem = StorageStatusElement(ss._storage, ss._nickname) return flattenString(None, elem) -@implementer(IRequest) -class JSONRequest(DummyRequest): - """ - A Request with t=json argument added to it. - - This is useful to invoke a Resouce.render_JSON() method. - """ - def __init__(self, **kwargs): - DummyRequest.__init__(self, b"/", **kwargs) - self.args = {"t": ["json"]} - self.fields = {} - def renderJSON(resource): - """Exercise resouce.render_JSON() + """Render a JSON from the given resource.""" + + @implementer(IRequest) + class JSONRequest(Request): + """ + A Request with t=json argument added to it. This is useful to + invoke a Resouce.render_JSON() method. + """ + def __init__(self): + Request.__init__(self, DummyChannel()) + self.args = {"t": ["json"]} + self.fields = {} - :param _MultiFormatResource resouce: A `twisted.web.resouce.Resource` - that contains a render_JSON() method. - """ return resource.render(JSONRequest()) class MyBucketCountingCrawler(BucketCountingCrawler): From aac8dee8a0dd0aeab9724800c306b2538bbcae1f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:07:47 -0400 Subject: [PATCH 042/134] Use twisted template tags in upload status page template --- src/allmydata/web/upload-status.xhtml | 62 +++++++++++++-------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/upload-status.xhtml b/src/allmydata/web/upload-status.xhtml index 8aaecd47a..95a5f7692 100644 --- a/src/allmydata/web/upload-status.xhtml +++ b/src/allmydata/web/upload-status.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Upload Status @@ -10,46 +10,46 @@

        File Upload Status

          -
        • Started:
          • -
        • Storage Index:
          • -
        • Helper?:
          • -
        • Total Size:
          • -
        • Progress (Hash):
          • -
        • Progress (Ciphertext):
          • -
        • Progress (Encode+Push):
          • -
        • Status:
          • +
        • Started:
          • +
        • Storage Index:
          • +
        • Helper?:
          • +
        • Total Size:
          • +
        • Progress (Hash):
          • +
        • Progress (Ciphertext):
          • +
        • Progress (Encode+Push):
          • +
        • Status:
        -
        +

        Upload Results

          -
        • Shares Pushed:
          • -
        • Shares Already Present:
          • -
        • Sharemap:
          • -
        • Servermap:
          • +
        • Shares Pushed:
          • +
        • Shares Already Present:
          • +
        • Sharemap:
          • +
        • Servermap:
        • Timings:
            • -
          • File Size: bytes
            • -
          • Total: - ()
            • +
          • File Size: bytes
            • +
          • Total: + ()
              • -
            • Storage Index: - ()
              • -
            • [Contacting Helper]:
              • -
            • [Upload Ciphertext To Helper]: - ()
              • +
            • Storage Index: + ()
              • +
            • [Contacting Helper]:
              • +
            • [Upload Ciphertext To Helper]: + ()
              • -
            • Peer Selection:
              • -
            • Encode And Push: - ()
              • +
            • Peer Selection:
              • +
            • Encode And Push: + ()
                • -
              • Cumulative Encoding: - ()
                • -
              • Cumulative Pushing: - ()
                • -
              • Send Hashes And Close:
                • +
              • Cumulative Encoding: + ()
                • +
              • Cumulative Pushing: + ()
                • +
              • Send Hashes And Close:
              -
            • [Helper Total]:
              • +
            • [Helper Total]:
        From 41300097c8db4b8ad6c3ed709c1fd78b0eb3d2ec Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:08:31 -0400 Subject: [PATCH 043/134] Make UploadStatusPage a MultiFormatResource --- src/allmydata/web/status.py | 85 +++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index b20aa679e..2c284caf6 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -160,62 +160,85 @@ class UploadResultsRendererMixin(RateAndTimeMixin): d.addCallback(_convert) return d -class UploadStatusPage(UploadResultsRendererMixin, rend.Page): - docFactory = getxmlfile("upload-status.xhtml") - def __init__(self, data): - rend.Page.__init__(self, data) - self.upload_status = data +class UploadStatusPage(MultiFormatResource): + + def __init__(self, upload_status): + super(UploadStatusPage, self).__init__() + self._upload_status = upload_status + + def render_HTML(self, req): + elem = UploadStatusElement(self._upload_status) + return renderElement(req, elem) + + +class UploadStatusElement(Element, UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) + + def __init__(self, upload_status): + super(UploadStatusElement, self).__init__() + self._upload_status = upload_status def upload_results(self): - return defer.maybeDeferred(self.upload_status.get_results) + return defer.maybeDeferred(self._upload_status.get_results) - def render_results(self, ctx, data): + @renderer + def results(self, req, tag): d = self.upload_results() def _got_results(results): if results: - return ctx.tag + return tag return "" d.addCallback(_got_results) return d - def render_started(self, ctx, data): - started_s = render_time(data.get_started()) - return started_s + @renderer + def started(self, req, tag): + started_s = render_time(self._upload_status.get_started()) + return tag(started_s) - def render_si(self, ctx, data): - si_s = base32.b2a_or_none(data.get_storage_index()) + @renderer + def si(self, req, tag): + si_s = base32.b2a_or_none(self._upload_status.get_storage_index()) if si_s is None: si_s = "(None)" - return si_s + return tag(str(si_s)) - def render_helper(self, ctx, data): - return {True: "Yes", - False: "No"}[data.using_helper()] + @renderer + def helper(self, req, tag): + return tag({True: "Yes", + False: "No"}[self._upload_status.using_helper()]) - def render_total_size(self, ctx, data): - size = data.get_size() + @renderer + def total_size(self, req, tag): + size = self._upload_status.get_size() if size is None: return "(unknown)" - return size + return tag(str(size)) - def render_progress_hash(self, ctx, data): - progress = data.get_progress()[0] + @renderer + def progress_hash(self, req, tag): + progress = self._upload_status.get_progress()[0] + # TODO: make an ascii-art bar + return tag("%.1f%%" % (100.0 * progress)) + + @renderer + def progress_ciphertext(self, req, tag): + progress = self._upload_status.get_progress()[1] # TODO: make an ascii-art bar return "%.1f%%" % (100.0 * progress) - def render_progress_ciphertext(self, ctx, data): - progress = data.get_progress()[1] + @renderer + def progress_encode_push(self, req, tag): + progress = self._upload_status.get_progress()[2] # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + return tag("%.1f%%" % (100.0 * progress)) - def render_progress_encode_push(self, ctx, data): - progress = data.get_progress()[2] - # TODO: make an ascii-art bar - return "%.1f%%" % (100.0 * progress) + @renderer + def status(self, req, tag): + return tag(self._upload_status.get_status()) - def render_status(self, ctx, data): - return data.get_status() class DownloadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'download_results' From 647071869173646a2567a9ba844bd8bbdd5884bb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:32 -0400 Subject: [PATCH 044/134] Use twisted template tags in upload results page template --- src/allmydata/web/upload-results.xhtml | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/allmydata/web/upload-results.xhtml b/src/allmydata/web/upload-results.xhtml index fa8894dcd..290c9bd42 100644 --- a/src/allmydata/web/upload-results.xhtml +++ b/src/allmydata/web/upload-results.xhtml @@ -1,4 +1,4 @@ - + Tahoe-LAFS - File Uploaded @@ -7,37 +7,37 @@ -

        Uploading File...

        +

        Uploading File...

        Upload Results:

          -
        • URI:
          • -
        • Download link:
          • -
        • Sharemap:
          • -
        • Servermap:
          • +
        • URI:
          • +
        • Download link:
          • +
        • Sharemap:
          • +
        • Servermap:
        • Timings:
            • -
          • File Size: bytes
            • -
          • Total: - ()
            • +
          • File Size: bytes
            • +
          • Total: + ()
              • -
            • Storage Index: - ()
              • -
            • [Contacting Helper]:
              • -
            • [Upload Ciphertext To Helper]: - ()
              • +
            • Storage Index: + ()
              • +
            • [Contacting Helper]:
              • +
            • [Upload Ciphertext To Helper]: + ()
              • -
            • Peer Selection:
              • -
            • Encode And Push: - ()
              • +
            • Peer Selection:
              • +
            • Encode And Push: + ()
                • -
              • Cumulative Encoding: - ()
                • -
              • Cumulative Pushing: - ()
                • -
              • Send Hashes And Close:
                • +
              • Cumulative Encoding: + ()
                • +
              • Cumulative Pushing: + ()
                • +
              • Send Hashes And Close:
              -
            • [Helper Total]:
              • +
            • [Helper Total]:
        From 57b1203d17a713ad58fc5b2e0698ba073f0ce349 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:50:54 -0400 Subject: [PATCH 045/134] Make UploadResultsPage a MultiFormatResource --- src/allmydata/web/unlinked.py | 58 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 8a84a4021..0abf6cc6d 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -2,11 +2,27 @@ import urllib from twisted.web import http from twisted.internet import defer -from nevow import rend, url, tags as T +from twisted.python.filepath import FilePath +from twisted.web.template import ( + Element, + XMLFile, + renderer, + renderElement, + tags, +) +from nevow import url, tags as T from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle -from allmydata.web.common import getxmlfile, get_arg, boolean_of_arg, \ - convert_children_json, WebError, get_format, get_mutable_type +from allmydata.web.common import ( + getxmlfile, + get_arg, + boolean_of_arg, + convert_children_json, + WebError, + get_format, + get_mutable_type, + MultiFormatResource, +) from allmydata.web import status def PUTUnlinkedCHK(req, client): @@ -59,34 +75,50 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(status.UploadResultsRendererMixin, rend.Page): +class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" - docFactory = getxmlfile("upload-results.xhtml") def __init__(self, upload_results): - rend.Page.__init__(self) - self.results = upload_results + super(UploadResultsPage, self).__init__() + self._upload_results = upload_results + + def render_HTML(self, req): + elem = UploadResultsElement(self._upload_results) + return renderElement(req, elem) + + +class UploadResultsElement(Element, status.UploadResultsRendererMixin): + + loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) + + def __init__(self, upload_results): + super(UploadResultsElement, self).__init__() + self._upload_results = upload_results def upload_results(self): - return defer.succeed(self.results) + return defer.succeed(self._upload_results) - def data_done(self, ctx, data): + @renderer + def done(self, req, tag): d = self.upload_results() d.addCallback(lambda res: "done!") return d - def data_uri(self, ctx, data): + @renderer + def uri(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_uri()) return d - def render_download_link(self, ctx, data): + @renderer + def download_link(self, req, tag): d = self.upload_results() d.addCallback(lambda res: - T.a(href="/uri/" + urllib.quote(res.get_uri())) - ["/uri/" + res.get_uri()]) + tags.a("/uri/" + res.get_uri(), + href="/uri/" + urllib.quote(res.get_uri()))) return d + def POSTUnlinkedSSK(req, client, version): # "POST /uri", to create an unlinked file. # SDMF: files are small, and we can only upload data From 3bfa5a5c292c33ff7eb4679938863aa039257d19 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 17:53:38 -0400 Subject: [PATCH 046/134] Use twisted web template in UploadResultsRendererMixin --- src/allmydata/web/status.py | 120 +++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 49 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2c284caf6..4edc63234 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -35,128 +35,150 @@ class RateAndTimeMixin(object): def render_rate(self, ctx, data): return abbreviate_rate(data) + class UploadResultsRendererMixin(RateAndTimeMixin): # this requires a method named 'upload_results' - def render_pushed_shares(self, ctx, data): + @renderer + def pushed_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_pushed_shares()) + d.addCallback(lambda res: str(res.get_pushed_shares())) return d - def render_preexisting_shares(self, ctx, data): + @renderer + def preexisting_shares(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_preexisting_shares()) + d.addCallback(lambda res: str(res.get_preexisting_shares())) return d - def render_sharemap(self, ctx, data): + @renderer + def sharemap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_sharemap()) def _render(sharemap): if sharemap is None: return "None" - l = T.ul() + ul = tags.ul() for shnum, servers in sorted(sharemap.items()): server_names = ', '.join([s.get_name() for s in servers]) - l[T.li["%d -> placed on [%s]" % (shnum, server_names)]] - return l + ul(tags.li("%d -> placed on [%s]" % (shnum, server_names))) + return ul d.addCallback(_render) return d - def render_servermap(self, ctx, data): + @renderer + def servermap(self, req, tag): d = self.upload_results() d.addCallback(lambda res: res.get_servermap()) def _render(servermap): if servermap is None: return "None" - l = T.ul() + ul = tags.ul() for server, shnums in sorted(servermap.items()): shares_s = ",".join(["#%d" % shnum for shnum in shnums]) - l[T.li["[%s] got share%s: %s" % (server.get_name(), - plural(shnums), shares_s)]] - return l + ul(tags.li("[%s] got share%s: %s" % (server.get_name(), + plural(shnums), shares_s))) + return ul d.addCallback(_render) return d - def data_file_size(self, ctx, data): + @renderer + def file_size(self, req, tag): d = self.upload_results() - d.addCallback(lambda res: res.get_file_size()) + d.addCallback(lambda res: str(res.get_file_size())) return d def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: res.get_timings().get(name)) + d.addCallback(lambda res: str(res.get_timings().get(name))) return d - def data_time_total(self, ctx, data): - return self._get_time("total") + @renderer + def time_total(self, req, tag): + return tag(self._get_time("total")) - def data_time_storage_index(self, ctx, data): - return self._get_time("storage_index") + @renderer + def time_storage_index(self, req, tag): + return tag(self._get_time("storage_index")) - def data_time_contacting_helper(self, ctx, data): - return self._get_time("contacting_helper") + @renderer + def time_contacting_helper(self, req, tag): + return tag(self._get_time("contacting_helper")) - def data_time_cumulative_fetch(self, ctx, data): - return self._get_time("cumulative_fetch") + @renderer + def time_cumulative_fetch(self, req, tag): + return tag(self._get_time("cumulative_fetch")) - def data_time_helper_total(self, ctx, data): - return self._get_time("helper_total") + @renderer + def time_helper_total(self, req, tag): + return tag(self._get_time("helper_total")) - def data_time_peer_selection(self, ctx, data): - return self._get_time("peer_selection") + @renderer + def time_peer_selection(self, req, tag): + return tag(self._get_time("peer_selection")) - def data_time_total_encode_and_push(self, ctx, data): - return self._get_time("total_encode_and_push") + @renderer + def time_total_encode_and_push(self, req, tag): + return tag(self._get_time("total_encode_and_push")) - def data_time_cumulative_encoding(self, ctx, data): - return self._get_time("cumulative_encoding") + @renderer + def time_cumulative_encoding(self, req, tag): + return tag(self._get_time("cumulative_encoding")) - def data_time_cumulative_sending(self, ctx, data): - return self._get_time("cumulative_sending") + @renderer + def time_cumulative_sending(self, req, tag): + return tag(self._get_time("cumulative_sending")) - def data_time_hashes_and_close(self, ctx, data): - return self._get_time("hashes_and_close") + @renderer + def time_hashes_and_close(self, req, tag): + return tag(self._get_time("hashes_and_close")) def _get_rate(self, name): d = self.upload_results() def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return compute_rate(file_size, duration) + return str(compute_rate(file_size, duration)) d.addCallback(_convert) return d - def data_rate_total(self, ctx, data): - return self._get_rate("total") + @renderer + def rate_total(self, req, tag): + return tag(self._get_rate("total")) - def data_rate_storage_index(self, ctx, data): - return self._get_rate("storage_index") + @renderer + def rate_storage_index(self, req, tag): + return tag(self._get_rate("storage_index")) - def data_rate_encode(self, ctx, data): - return self._get_rate("cumulative_encoding") + @renderer + def rate_encode(self, req, tag): + return tag(self._get_rate("cumulative_encoding")) - def data_rate_push(self, ctx, data): + @renderer + def rate_push(self, req, tag): return self._get_rate("cumulative_sending") - def data_rate_encode_and_push(self, ctx, data): + @renderer + def rate_encode_and_push(self, req, tag): d = self.upload_results() def _convert(r): file_size = r.get_file_size() time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return None + return str(None) else: - return compute_rate(file_size, time1+time2) + return str(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d - def data_rate_ciphertext_fetch(self, ctx, data): + @renderer + def rate_ciphertext_fetch(self, req, tag): d = self.upload_results() def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return compute_rate(fetch_size, duration) + return str(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From 7a82fd671f4377c569957beb234a8c5cc55254b9 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 May 2020 18:00:20 -0400 Subject: [PATCH 047/134] Handle "t=upload" in UploadResultsPage POST handling --- src/allmydata/web/unlinked.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 0abf6cc6d..283d8d586 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -86,6 +86,23 @@ class UploadResultsPage(MultiFormatResource): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) + # This is weird but necessary because: + # + # 1. MultiFormatResource.render() uses argument "t" to figure out + # its output format. + # + # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". + # See URIHandler.render_POST(). + # + # MultiFormatResource.render() looks up "t" argument, which in + # this case has the value "upload", and then it would look for a + # render_UPLOAD() method. + # + # We could change upload request to use more descriptive names + # that do not cause name collisions like this. That should be a + # separate change though. + render_UPLOAD = render_HTML + class UploadResultsElement(Element, status.UploadResultsRendererMixin): From 54ac2d224ae449a8da651139a3f60a80de0ed664 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:23:23 -0400 Subject: [PATCH 048/134] Remove unused imports --- src/allmydata/web/unlinked.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 283d8d586..33fb7bafa 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -10,11 +10,10 @@ from twisted.web.template import ( renderElement, tags, ) -from nevow import url, tags as T +from nevow import url from allmydata.immutable.upload import FileHandle from allmydata.mutable.publish import MutableFileHandle from allmydata.web.common import ( - getxmlfile, get_arg, boolean_of_arg, convert_children_json, From 5fdd61b1464f35b2a5df056d04540eab599917d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:25:09 -0400 Subject: [PATCH 049/134] Document UploadStatusPage --- src/allmydata/web/status.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4edc63234..be87cff65 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -184,8 +184,12 @@ class UploadResultsRendererMixin(RateAndTimeMixin): class UploadStatusPage(MultiFormatResource): + """Renders /status/up-%d.""" def __init__(self, upload_status): + """ + :param IUploadStatus upload_status: stats provider. + """ super(UploadStatusPage, self).__init__() self._upload_status = upload_status From 7be18839104b025d78e00f9105714936455a7df4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 13:28:16 -0400 Subject: [PATCH 050/134] Document UploadResultsPage parameter --- src/allmydata/web/unlinked.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 33fb7bafa..6ca219c8b 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -78,6 +78,9 @@ class UploadResultsPage(MultiFormatResource): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): + """ + :param IUploadResults upload_results: stats provider. + """ super(UploadResultsPage, self).__init__() self._upload_results = upload_results From aecd90858a852e242e990729e97f9d9a5af22c51 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 14:53:33 -0400 Subject: [PATCH 051/134] Render abbreviated rate and time values --- src/allmydata/web/status.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index be87cff65..61ab0ae26 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -90,7 +90,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _get_time(self, name): d = self.upload_results() - d.addCallback(lambda res: str(res.get_timings().get(name))) + d.addCallback(lambda res: abbreviate_time(res.get_timings().get(name))) return d @renderer @@ -138,7 +138,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): file_size = r.get_file_size() duration = r.get_timings().get(name) - return str(compute_rate(file_size, duration)) + return abbreviate_rate(compute_rate(file_size, duration)) d.addCallback(_convert) return d @@ -166,9 +166,9 @@ class UploadResultsRendererMixin(RateAndTimeMixin): time1 = r.get_timings().get("cumulative_encoding") time2 = r.get_timings().get("cumulative_sending") if (time1 is None or time2 is None): - return str(None) + return abbreviate_rate(None) else: - return str(compute_rate(file_size, time1+time2)) + return abbreviate_rate(compute_rate(file_size, time1+time2)) d.addCallback(_convert) return d @@ -178,7 +178,7 @@ class UploadResultsRendererMixin(RateAndTimeMixin): def _convert(r): fetch_size = r.get_ciphertext_fetched() duration = r.get_timings().get("cumulative_fetch") - return str(compute_rate(fetch_size, duration)) + return abbreviate_rate(compute_rate(fetch_size, duration)) d.addCallback(_convert) return d From 00f70f6539f3360806feca100802fb8deabc16f3 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:08:34 -0400 Subject: [PATCH 052/134] Add newsfragment --- newsfragments/3287.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3287.minor diff --git a/newsfragments/3287.minor b/newsfragments/3287.minor new file mode 100644 index 000000000..e69de29bb From 91f8d939f8dd5510c2542178f0326800d66a51f0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:22:59 -0400 Subject: [PATCH 053/134] Avoid use of RateAndTimeMixin in UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 61ab0ae26..eb7fedc6f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -36,7 +36,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(RateAndTimeMixin): +class UploadResultsRendererMixin(object): # this requires a method named 'upload_results' @renderer @@ -383,7 +383,7 @@ class DownloadResultsRendererMixin(RateAndTimeMixin): l = T.ul() for peerid in sorted(per_server.keys()): peerid_s = idlib.shortnodeid_b2a(peerid) - times_s = ", ".join([self.render_time(None, t) + times_s = ", ".join([abbreviate_time(t) for t in per_server[peerid]]) l[T.li["[%s]: %s" % (peerid_s, times_s)]] return T.li["Per-Server Segment Fetch Response Times: ", l] From b0e3325d03e6af3d2bd1556a91335ef8767b895a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 20:49:34 -0400 Subject: [PATCH 054/134] Move time formatter tests UploadResultsRendererMixin no longer has render_time(); moving tests for render_time() as tests for abbreviate_time() to a more appropriate place. --- src/allmydata/test/web/test_util.py | 10 ++++++++++ src/allmydata/test/web/test_web.py | 11 ----------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/allmydata/test/web/test_util.py b/src/allmydata/test/web/test_util.py index 2a6f67abc..a378c1447 100644 --- a/src/allmydata/test/web/test_util.py +++ b/src/allmydata/test/web/test_util.py @@ -23,6 +23,16 @@ class Util(ShouldFailMixin, testutil.ReallyEqualMixin, unittest.TestCase): self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") self.failUnlessReallyEqual(common.abbreviate_time(-123000), "-123000000000us") + self.failUnlessReallyEqual(common.abbreviate_time(None), "") + self.failUnlessReallyEqual(common.abbreviate_time(2.5), "2.50s") + self.failUnlessReallyEqual(common.abbreviate_time(0.25), "250ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.0021), "2.1ms") + self.failUnlessReallyEqual(common.abbreviate_time(0.000123), "123us") + self.failUnlessReallyEqual(common.abbreviate_rate(None), "") + self.failUnlessReallyEqual(common.abbreviate_rate(2500000), "2.50MBps") + self.failUnlessReallyEqual(common.abbreviate_rate(30100), "30.1kBps") + self.failUnlessReallyEqual(common.abbreviate_rate(123), "123Bps") + def test_compute_rate(self): self.failUnlessReallyEqual(common.compute_rate(None, None), None) self.failUnlessReallyEqual(common.compute_rate(None, 1), None) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 5be0b2f7b..d7137ad11 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1046,17 +1046,6 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi self.failUnlessReallyEqual(drrm.render_rate(None, 30100), "30.1kBps") self.failUnlessReallyEqual(drrm.render_rate(None, 123), "123Bps") - urrm = status.UploadResultsRendererMixin() - self.failUnlessReallyEqual(urrm.render_time(None, None), "") - self.failUnlessReallyEqual(urrm.render_time(None, 2.5), "2.50s") - self.failUnlessReallyEqual(urrm.render_time(None, 0.25), "250ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.0021), "2.1ms") - self.failUnlessReallyEqual(urrm.render_time(None, 0.000123), "123us") - self.failUnlessReallyEqual(urrm.render_rate(None, None), "") - self.failUnlessReallyEqual(urrm.render_rate(None, 2500000), "2.50MBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 30100), "30.1kBps") - self.failUnlessReallyEqual(urrm.render_rate(None, 123), "123Bps") - def test_GET_FILEURL(self): d = self.GET(self.public_url + "/foo/bar.txt") d.addCallback(self.failUnlessIsBarDotTxt) From f8562530be3d6c0a07b180a74558b818fadd8ffb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:11:42 -0400 Subject: [PATCH 055/134] Make Element the superclass of UploadResultsRendererMixin --- src/allmydata/web/status.py | 4 ++-- src/allmydata/web/unlinked.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index eb7fedc6f..8147437fa 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -36,7 +36,7 @@ class RateAndTimeMixin(object): return abbreviate_rate(data) -class UploadResultsRendererMixin(object): +class UploadResultsRendererMixin(Element): # this requires a method named 'upload_results' @renderer @@ -198,7 +198,7 @@ class UploadStatusPage(MultiFormatResource): return renderElement(req, elem) -class UploadStatusElement(Element, UploadResultsRendererMixin): +class UploadStatusElement(UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-status.xhtml")) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 6ca219c8b..d938691a4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -106,7 +106,7 @@ class UploadResultsPage(MultiFormatResource): render_UPLOAD = render_HTML -class UploadResultsElement(Element, status.UploadResultsRendererMixin): +class UploadResultsElement(status.UploadResultsRendererMixin): loader = XMLFile(FilePath(__file__).sibling("upload-results.xhtml")) From 03d529c22a91020b893fab8c28678918f3450b6d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 22 May 2020 21:33:30 -0400 Subject: [PATCH 056/134] Remove unused import --- src/allmydata/web/unlinked.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index d938691a4..91efff2c4 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -4,7 +4,6 @@ from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.web.template import ( - Element, XMLFile, renderer, renderElement, From d50ca9af55c2a19ccf35c4ff0b128ecafb75331f Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 20 May 2020 22:43:21 -0600 Subject: [PATCH 057/134] proof-of-concept verified-fake for testing --- integration/test_testing.py | 26 +++ src/allmydata/testing/__init__.py | 0 src/allmydata/testing/web.py | 263 ++++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+) create mode 100644 integration/test_testing.py create mode 100644 src/allmydata/testing/__init__.py create mode 100644 src/allmydata/testing/web.py diff --git a/integration/test_testing.py b/integration/test_testing.py new file mode 100644 index 000000000..47498b1d7 --- /dev/null +++ b/integration/test_testing.py @@ -0,0 +1,26 @@ + + +from allmydata.testing.web import ( + create_fake_tahoe_root, + deterministic_key_generator, +) + +import pytest +import pytest_twisted + + +@pytest_twisted.inlineCallbacks +def test_retrieve_cap(): + """ + WebUI Fake can serve a read-capability back + """ + + keys = deterministic_key_generator() + root = yield create_fake_tahoe_root() + dummy_readcap = yield root.add_data( + next(keys), + "some dummy content\n"*20 + ) + print("readcap: {}".format(dummy_readcap)) + + assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" diff --git a/src/allmydata/testing/__init__.py b/src/allmydata/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py new file mode 100644 index 000000000..5cf517a54 --- /dev/null +++ b/src/allmydata/testing/web.py @@ -0,0 +1,263 @@ +import io +import os +import string + +import attr + +from twisted.internet import defer +from twisted.python.filepath import ( + FilePath, +) +from twisted.web.resource import ( + Resource, +) +from twisted.web.client import ( + Agent, + FileBodyProducer, +) + +from treq.client import ( + HTTPClient, +) +from treq.testing import ( + RequestTraversalAgent, + RequestSequence, + StubTreq, +) + + +class _FakeTahoeRoot(Resource): + """ + This is a sketch of how an in-memory 'fake' of a Tahoe + WebUI. Ultimately, this will live in Tahoe + """ + + def __init__(self, uri=None): + Resource.__init__(self) # this is an old-style class :( + self._uri = uri + self.putChild(b"uri", self._uri) + + def add_data(self, key, data): + return self._uri.add_data(key, data) + +@attr.s +class _FakeCapability(object): + """ + """ + data=attr.ib() + + +# XXX want to make all kinds of caps, like +# URI:CHK:... URI:DIR2:... etc + +import allmydata.uri +KNOWN_CAPABILITIES = [ + getattr(allmydata.uri, t).BASE_STRING + for t in dir(allmydata.uri) + if hasattr(getattr(allmydata.uri, t), 'BASE_STRING') +] + + +from allmydata.immutable.upload import BaseUploadable +from allmydata.interfaces import IUploadable +from zope.interface import implementer +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) + + +def deterministic_key_generator(): + character = 0 + while character < (26 * 2): + key = string.letters[character] * 16 + character += 1 + yield key + raise RuntimeError("Ran out of keys") + + +@implementer(IUploadable) +class DataUploadable(BaseUploadable): + # Base gives us: + # set_upload_status + # set_default_encoding_parameters + # get_all_encoding_parameters + + def __init__(self, data, key=None): + self._data = data + self._where = 0 + self._key = key if key is not None else urandom(16) + + def get_encryption_key(self): + return succeed(self._key) + + def get_size(self): + return succeed(len(self._data)) + + @inlineCallbacks + def read(self, amount): + data = [self._data[self._where : self._where + amount]] + self._where += amount + yield + returnValue(data) + + def close(self): + pass + +@inlineCallbacks +def create_fake_capability(kind, key, data): + if kind not in KNOWN_CAPABILITIES: + raise ValueError( + "'{}' not a known kind: {}".format( + kind, + ", ".join(list(KNOWN_CAPABILITIES.keys())), + ) + ) + + # XXX to use a allmydata.immutable.upload.CHKUploader directly, + # we'd need to instantiate: + + from allmydata.immutable.upload import ( + CHKUploader, + EncryptAnUploadable, + ) + from allmydata.immutable.encode import ( + Encoder, + ) + + class _FakeSecretHolder(object): + def get_renewal_secret(self): + return "renewal_secret" + + def get_cancel_secret(self): + return "cancel_secret" + + + @attr.s + class _FakeBucket(object): + data = attr.ib(init=False, default="") + + def callRemoteOnly(self, *args): + pass # print("callRemoteOnly({})".format(args)) + + def callRemote(self, verb, *args): + if verb == 'write': + offset, data = args + assert offset >= len(self.data) + while offset > len(self.data): + self.data += 'X' # marker data; we're padding + self.data += data + elif verb == 'close': + pass + else: + print("callRemote({})".format(args)) + + + @attr.s + class _FakeStorageServer(object): + buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) + + def get_buckets(self, storage_index): + return succeed(self.buckets) + + def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): + # returns a 2-tuple .. second one maps share-num to BucketWriter + return succeed(( + {}, + { + i: bucket + for i, bucket in enumerate(self.buckets) + } + )) + + class _FakeServer(object): + def get_serverid(self): + return "fake_server" + + def get_name(self): + return "steven" + + def get_version(self): + return { + "http://allmydata.org/tahoe/protocols/storage/v1": { + "maximum-immutable-share-size": 10*1024*1024*1024, + } + } + + def get_lease_seed(self): + return "decafbaddecafbaddeca" + + def get_storage_server(self): + return _FakeStorageServer() + + + class _FakeStorageBroker(object): + def get_servers_for_psi(self, storage_index): + return [_FakeServer()] + + storage_broker = _FakeStorageBroker() + secret_holder = _FakeSecretHolder() + uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) + uploadable = DataUploadable(data, key=key) + uploadable.set_default_encoding_parameters({ + "n": 1, + "k": 1, + "happy": 1, + }) + encrypted_uploadable = EncryptAnUploadable(uploadable) + + encoder = Encoder() + yield encoder.set_encrypted_uploadable(encrypted_uploadable) + + uploadresults = yield uploader.start(encrypted_uploadable) + + enc_key = yield uploadable.get_encryption_key() + + from allmydata.uri import from_string as uri_from_string + from allmydata.uri import CHKFileURI + + verify_cap = uri_from_string(uploadresults.get_verifycapstr()) + read_cap = CHKFileURI( + enc_key, + verify_cap.uri_extension_hash, + verify_cap.needed_shares, + verify_cap.total_shares, + verify_cap.size, + ) + uploadresults.set_uri(read_cap.to_string()) + + returnValue(read_cap) + + +class _FakeTahoeUriHandler(Resource): + """ + """ + + isLeaf = True + + @inlineCallbacks + def add_data(self, key, data): + """ + adds some data to our grid, returning a capability + """ + cap = yield create_fake_capability("URI:CHK:", key, data) + returnValue(cap) + + def render_GET(self, request): + print(request) + print(request.uri) + return b"URI:DIR2-CHK:some capability" + + +@inlineCallbacks +def create_fake_tahoe_root(): + """ + Probably should take some params to control what this fake does: + return errors, pre-populate capabilities, ... + """ + root = _FakeTahoeRoot( + uri=_FakeTahoeUriHandler(), + ) + yield + returnValue(root) From cf4b3ba008a2ac760c39899570041dfb2bd8e304 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:42:55 -0400 Subject: [PATCH 058/134] Correct arguments to super() This was incidentally found when looking at some test failures: super(type, object) is the right invocation, and not the other way around. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index e7a94ff40..a2af857b9 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1269,5 +1269,5 @@ class TrialTestCase(_TrialTestCase): if six.PY2: if isinstance(msg, six.text_type): - return super(self, TrialTestCase).fail(msg.encode("utf8")) - return super(self, TrialTestCase).fail(msg) + return super(TrialTestCase, self).fail(msg.encode("utf8")) + return super(TrialTestCase, self).fail(msg) From b6e00d09f9eee5e6c829b91b3244b6180d239c94 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Thu, 28 May 2020 13:49:13 -0400 Subject: [PATCH 059/134] Render self in DirectoryNodeHandler.getChild() We need self.render_POST() etc. to be invoked when we have a request such as "POST /uri/URI:DIR:..."; throwing an error here is probably not the right thing to do. --- src/allmydata/web/directory.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 57c222788..2129cec25 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -103,12 +103,10 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page - name = name.decode('utf8') if not name: - raise EmptyPathnameComponentError( - u"The webapi does not allow empty pathname components", - ) + return self + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From ae7802fc0c5481f51bdc37ec63025191215b30eb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 09:24:47 -0400 Subject: [PATCH 060/134] Add newsfragment --- newsfragments/3312.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3312.bugfix diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3312.bugfix @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From 0633623eccf113a52f719f56db517b2ff7e0808d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 19:47:12 -0400 Subject: [PATCH 061/134] Reject URIs that contain empty segments --- src/allmydata/web/directory.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index 2129cec25..fced8c68e 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -106,6 +106,16 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): if not name: return self + # Rejecting URIs that contain empty path pieces (for example: + # "/uri/URI:DIR2:../foo//new.txt" or "/uri/URI:DIR2:..//") was + # the old nevow behavior and it is encoded in the test suite; + # we will follow suit. + for segment in req.prepath: + if not segment: + raise EmptyPathnameComponentError( + u"The webapi does not allow empty pathname components", + ) + name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) From 31eba00d0fef88d6ba2aad6b28cadda94ff50bd1 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:17 -0400 Subject: [PATCH 062/134] Reject trailing empty path piece in URIs --- src/allmydata/web/directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index fced8c68e..b1618a3f5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -102,8 +102,9 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # trying to replicate what I have observed as Nevow behavior # for these nodes, which is that a URI like # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash - # or no further children) renders "this" page - if not name: + # or no further children) renders "this" page. We also need + # to reject "/uri/URI:DIR2:..//", so we look at postpath. + if not name and req.postpath != ['']: return self # Rejecting URIs that contain empty path pieces (for example: From dce73f7c2fd4c304782342a1bb51645b62a87a53 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:38:35 -0400 Subject: [PATCH 063/134] Make "Refresh" link work This seems to be a subtle difference from nevow: with `href="."`, rendered link target will be `/uri/`, so clicking "Refresh" will result in an error message like so: "GET /uri requires uri=". With `href=""`, the rendered link target will be `/uri/URI:...`, which is what we need. --- src/allmydata/web/directory.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.xhtml b/src/allmydata/web/directory.xhtml index 88058efa4..eeb04692d 100644 --- a/src/allmydata/web/directory.xhtml +++ b/src/allmydata/web/directory.xhtml @@ -31,7 +31,7 @@
        From dd14da4a55e043ab18d067c750ab20eb9050f5b4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 29 May 2020 21:46:23 -0400 Subject: [PATCH 064/134] Decode name before null check Make the diff against master a tad bit smaller. --- src/allmydata/web/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/directory.py b/src/allmydata/web/directory.py index b1618a3f5..a5ba3bed5 100644 --- a/src/allmydata/web/directory.py +++ b/src/allmydata/web/directory.py @@ -104,6 +104,7 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): # "/uri/URI%3ADIR2%3Aj...vq/" (that is, with a trailing slash # or no further children) renders "this" page. We also need # to reject "/uri/URI:DIR2:..//", so we look at postpath. + name = name.decode('utf8') if not name and req.postpath != ['']: return self @@ -117,7 +118,6 @@ class DirectoryNodeHandler(ReplaceMeMixin, Resource, object): u"The webapi does not allow empty pathname components", ) - name = name.decode('utf8') d = self.node.get(name) d.addBoth(self._got_child, req, name) return d From 1faa81c7c4b6b7ad9a2dd1e5d1a0851d13eb39f9 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 1 Jun 2020 09:06:46 -0600 Subject: [PATCH 065/134] actual usable agent --- src/allmydata/testing/web.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5cf517a54..1767cb114 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -261,3 +261,52 @@ def create_fake_tahoe_root(): ) yield returnValue(root) + + +@implementer(IBodyProducer) +class _SynchronousProducer(object): + """ + A partial implementation of an :obj:`IBodyProducer` which produces its + entire payload immediately. There is no way to access to an instance of + this object from :obj:`RequestTraversalAgent` or :obj:`StubTreq`, or even a + :obj:`Resource: passed to :obj:`StubTreq`. + + This does not implement the :func:`IBodyProducer.stopProducing` method, + because that is very difficult to trigger. (The request from + `RequestTraversalAgent` would have to be canceled while it is still in the + transmitting state), and the intent is to use `RequestTraversalAgent` to + make synchronous requests. + """ + + def __init__(self, body): + """ + Create a synchronous producer with some bytes. + """ + if not isinstance(body, bytes): + raise ValueError( + "'body' must be bytes" + ) + self.body = body + self.length = len(body) + + def startProducing(self, consumer): + """ + Immediately produce all data. + """ + consumer.write(self.body) + return succeed(None) + + +@inlineCallbacks +def create_tahoe_treq_client(root=None): + """ + """ + + if root is None: + root = yield create_fake_tahoe_root() + + client = HTTPClient( + agent=RequestTraversalAgent(root), + data_to_body_producer=_SynchronousProducer, + ) + returnValue(client) From 4fa2e6ad97e70e56bcedb6b77d8be292d128e6c6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 11:56:59 -0400 Subject: [PATCH 066/134] Use twisted web directives in web.operations.ReloadMixin --- src/allmydata/web/operations.py | 36 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py index 93de2169e..eb9321344 100644 --- a/src/allmydata/web/operations.py +++ b/src/allmydata/web/operations.py @@ -1,7 +1,11 @@ import time -from nevow import rend, url, tags as T +from nevow import rend, url from nevow.inevow import IRequest +from twisted.web.template import ( + renderer, + tags as T, +) from twisted.python.failure import Failure from twisted.internet import reactor, defer from twisted.web.http import NOT_FOUND @@ -122,31 +126,31 @@ class OphandleTable(rend.Page, service.Service): class ReloadMixin(object): REFRESH_TIME = 1*MINUTE - def render_refresh(self, ctx, data): + @renderer + def refresh(self, req, tag): if self.monitor.is_finished(): return "" # dreid suggests ctx.tag(**dict([("http-equiv", "refresh")])) # but I can't tell if he's joking or not - ctx.tag.attributes["http-equiv"] = "refresh" - ctx.tag.attributes["content"] = str(self.REFRESH_TIME) - return ctx.tag + tag.attributes["http-equiv"] = "refresh" + tag.attributes["content"] = str(self.REFRESH_TIME) + return tag - def render_reload(self, ctx, data): + @renderer + def reload(self, req, tag): if self.monitor.is_finished(): return "" - req = IRequest(ctx) # url.gethere would break a proxy, so the correct thing to do is # req.path[-1] + queryargs ophandle = req.prepath[-1] reload_target = ophandle + "?output=html" cancel_target = ophandle + "?t=cancel" - cancel_button = T.form(action=cancel_target, method="POST", - enctype="multipart/form-data")[ - T.input(type="submit", value="Cancel"), - ] + cancel_button = T.form(T.input(type="submit", value="Cancel"), + action=cancel_target, + method="POST", + enctype="multipart/form-data",) - return [T.h2["Operation still running: ", - T.a(href=reload_target)["Reload"], - ], - cancel_button, - ] + return (T.h2("Operation still running: ", + T.a("Reload", href=reload_target), + ), + cancel_button,) From dd3b567a8635201af8369c46b931ffa09a49ab28 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 2 Jun 2020 12:06:48 -0400 Subject: [PATCH 067/134] Add newsfragment --- newsfragments/3315.minor | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3315.minor diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor new file mode 100644 index 000000000..ed00922a6 --- /dev/null +++ b/newsfragments/3315.minor @@ -0,0 +1 @@ +Add your info here \ No newline at end of file From 3e973197ac339ee0ff9acc7f0892dc255bbdbae2 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 3 Jun 2020 11:32:22 -0600 Subject: [PATCH 068/134] synchronous-only up/download tahoe code to keep treq.testing happy --- src/allmydata/testing/web.py | 73 ++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 1767cb114..32fe0c084 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,10 +1,12 @@ import io import os +import json import string import attr -from twisted.internet import defer +from hyperlink import DecodedURL + from twisted.python.filepath import ( FilePath, ) @@ -15,6 +17,9 @@ from twisted.web.client import ( Agent, FileBodyProducer, ) +from twisted.web.iweb import ( + IBodyProducer, +) from treq.client import ( HTTPClient, @@ -40,6 +45,7 @@ class _FakeTahoeRoot(Resource): def add_data(self, key, data): return self._uri.add_data(key, data) + @attr.s class _FakeCapability(object): """ @@ -70,8 +76,8 @@ from twisted.internet.defer import ( def deterministic_key_generator(): character = 0 - while character < (26 * 2): - key = string.letters[character] * 16 + while True:#character < (26 * 2): + key = string.letters[character % 52] * 16 character += 1 yield key raise RuntimeError("Ran out of keys") @@ -159,7 +165,7 @@ def create_fake_capability(kind, key, data): buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) def get_buckets(self, storage_index): - return succeed(self.buckets) + return succeed({0: self.buckets}) def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): # returns a 2-tuple .. second one maps share-num to BucketWriter @@ -230,11 +236,34 @@ def create_fake_capability(kind, key, data): returnValue(read_cap) +def capability_generator(): + number = 0 + while True: + cap = u"URI:CHK:fake capability {}".format(number) + number += 1 + yield cap.encode("ascii") + + class _FakeTahoeUriHandler(Resource): """ """ isLeaf = True + _data = None + _key_generator = None + _capability_generator = None + + def _get_key(self): + if self._key_generator is None: + self._key_generator = deterministic_key_generator() + key = next(self._key_generator) + return key + + def _generate_cap(self): + if self._capability_generator is None: + self._capability_generator = capability_generator() + capability = next(self._capability_generator) + return capability @inlineCallbacks def add_data(self, key, data): @@ -242,12 +271,42 @@ class _FakeTahoeUriHandler(Resource): adds some data to our grid, returning a capability """ cap = yield create_fake_capability("URI:CHK:", key, data) + if self._data is None: + self._data = dict() + self._data[cap] = data returnValue(cap) + def render_PUT(self, request): + data = request.content.read() +# if len(data) == 0: +# raise RuntimeError("No empty data") + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap + + def render_POST(self, request): + kind = request.args[u"t"][0] + data = request.content.read() + + cap = self._generate_cap() + if self._data is None: + self._data = dict() + self._data[cap] = data + return cap +# if kind == 'mkdir-immutable': + def render_GET(self, request): - print(request) - print(request.uri) - return b"URI:DIR2-CHK:some capability" + uri = DecodedURL.from_text(request.uri.decode('utf8')) + # XXX FIXME + capability = uri.query[0][1] + + if self._data is None or capability not in self._data: + return u"No data for '{}'".format(capability).decode("ascii") + + return self._data[capability] @inlineCallbacks From d7288ee2381b030b415e86df0461872aa6e70d9e Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 5 Jun 2020 09:28:36 -0400 Subject: [PATCH 069/134] Update newsfragment --- newsfragments/3315.minor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3315.minor b/newsfragments/3315.minor index ed00922a6..0536c297a 100644 --- a/newsfragments/3315.minor +++ b/newsfragments/3315.minor @@ -1 +1 @@ -Add your info here \ No newline at end of file +Replace nevow with twisted.web in web.operations.ReloadMixin From fc2203e421b33a50e3ded99eff0583247d6faae8 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:26:09 -0600 Subject: [PATCH 070/134] simplify tahoe web fakes --- src/allmydata/testing/web.py | 287 ++++++++++------------------------- 1 file changed, 76 insertions(+), 211 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 32fe0c084..78d2b7931 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -2,6 +2,7 @@ import io import os import json import string +import hashlib import attr @@ -20,6 +21,11 @@ from twisted.web.client import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.internet.defer import ( + inlineCallbacks, + succeed, + returnValue, +) from treq.client import ( HTTPClient, @@ -29,6 +35,12 @@ from treq.testing import ( RequestSequence, StubTreq, ) +from zope.interface import implementer + +import allmydata.uri +from allmydata.util import ( + base32, +) class _FakeTahoeRoot(Resource): @@ -53,10 +65,6 @@ class _FakeCapability(object): data=attr.ib() -# XXX want to make all kinds of caps, like -# URI:CHK:... URI:DIR2:... etc - -import allmydata.uri KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -64,183 +72,46 @@ KNOWN_CAPABILITIES = [ ] -from allmydata.immutable.upload import BaseUploadable -from allmydata.interfaces import IUploadable -from zope.interface import implementer -from twisted.internet.defer import ( - inlineCallbacks, - succeed, - returnValue, -) +def capability_generator(kind): + """ + :param str kind: the kind of capability, like `URI:CHK` - -def deterministic_key_generator(): - character = 0 - while True:#character < (26 * 2): - key = string.letters[character % 52] * 16 - character += 1 - yield key - raise RuntimeError("Ran out of keys") - - -@implementer(IUploadable) -class DataUploadable(BaseUploadable): - # Base gives us: - # set_upload_status - # set_default_encoding_parameters - # get_all_encoding_parameters - - def __init__(self, data, key=None): - self._data = data - self._where = 0 - self._key = key if key is not None else urandom(16) - - def get_encryption_key(self): - return succeed(self._key) - - def get_size(self): - return succeed(len(self._data)) - - @inlineCallbacks - def read(self, amount): - data = [self._data[self._where : self._where + amount]] - self._where += amount - yield - returnValue(data) - - def close(self): - pass - -@inlineCallbacks -def create_fake_capability(kind, key, data): + :returns: a generator that yields new capablities of a particular + kind. + """ if kind not in KNOWN_CAPABILITIES: raise ValueError( - "'{}' not a known kind: {}".format( + "Unknown capability kind '{} (valid are {})'".format( kind, - ", ".join(list(KNOWN_CAPABILITIES.keys())), + ", ".join(KNOWN_CAPABILITIES), ) ) - - # XXX to use a allmydata.immutable.upload.CHKUploader directly, - # we'd need to instantiate: - - from allmydata.immutable.upload import ( - CHKUploader, - EncryptAnUploadable, - ) - from allmydata.immutable.encode import ( - Encoder, - ) - - class _FakeSecretHolder(object): - def get_renewal_secret(self): - return "renewal_secret" - - def get_cancel_secret(self): - return "cancel_secret" - - - @attr.s - class _FakeBucket(object): - data = attr.ib(init=False, default="") - - def callRemoteOnly(self, *args): - pass # print("callRemoteOnly({})".format(args)) - - def callRemote(self, verb, *args): - if verb == 'write': - offset, data = args - assert offset >= len(self.data) - while offset > len(self.data): - self.data += 'X' # marker data; we're padding - self.data += data - elif verb == 'close': - pass - else: - print("callRemote({})".format(args)) - - - @attr.s - class _FakeStorageServer(object): - buckets = attr.ib(default=attr.Factory(lambda: [_FakeBucket()])) - - def get_buckets(self, storage_index): - return succeed({0: self.buckets}) - - def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, canary=None): - # returns a 2-tuple .. second one maps share-num to BucketWriter - return succeed(( - {}, - { - i: bucket - for i, bucket in enumerate(self.buckets) - } - )) - - class _FakeServer(object): - def get_serverid(self): - return "fake_server" - - def get_name(self): - return "steven" - - def get_version(self): - return { - "http://allmydata.org/tahoe/protocols/storage/v1": { - "maximum-immutable-share-size": 10*1024*1024*1024, - } - } - - def get_lease_seed(self): - return "decafbaddecafbaddeca" - - def get_storage_server(self): - return _FakeStorageServer() - - - class _FakeStorageBroker(object): - def get_servers_for_psi(self, storage_index): - return [_FakeServer()] - - storage_broker = _FakeStorageBroker() - secret_holder = _FakeSecretHolder() - uploader = CHKUploader(storage_broker, secret_holder, progress=None, reactor=None) - uploadable = DataUploadable(data, key=key) - uploadable.set_default_encoding_parameters({ - "n": 1, - "k": 1, - "happy": 1, - }) - encrypted_uploadable = EncryptAnUploadable(uploadable) - - encoder = Encoder() - yield encoder.set_encrypted_uploadable(encrypted_uploadable) - - uploadresults = yield uploader.start(encrypted_uploadable) - - enc_key = yield uploadable.get_encryption_key() - - from allmydata.uri import from_string as uri_from_string - from allmydata.uri import CHKFileURI - - verify_cap = uri_from_string(uploadresults.get_verifycapstr()) - read_cap = CHKFileURI( - enc_key, - verify_cap.uri_extension_hash, - verify_cap.needed_shares, - verify_cap.total_shares, - verify_cap.size, - ) - uploadresults.set_uri(read_cap.to_string()) - - returnValue(read_cap) - - -def capability_generator(): + # what we do here is to start with empty hashers for the key and + # ueb_hash and repeatedly feed() them a zero byte on each + # iteration .. so the same sequence of capabilities will always be + # produced. We could add a seed= argument if we wanted to produce + # different sequences. number = 0 + key_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") + + # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: - cap = u"URI:CHK:fake capability {}".format(number) number += 1 + key_hasher.update("\x00") + ueb_hasher.update("\x00") + + key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes + ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes + + cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + kind=kind, + key=key, + ueb_hash=ueb_hash, + n=1, + k=1, + size=number * 1000, + ) yield cap.encode("ascii") @@ -250,53 +121,50 @@ class _FakeTahoeUriHandler(Resource): isLeaf = True _data = None - _key_generator = None - _capability_generator = None + _capability_generators = None - def _get_key(self): - if self._key_generator is None: - self._key_generator = deterministic_key_generator() - key = next(self._key_generator) - return key + def _generate_capability(self, kind): + """ + :param str kind: any valid capability-string type - def _generate_cap(self): - if self._capability_generator is None: - self._capability_generator = capability_generator() - capability = next(self._capability_generator) + :returns: the next capability-string for the given kind + """ + if self._capability_generators is None: + self._capability_generators = dict() + + if kind not in self._capability_generators: + self._capability_generators[kind] = capability_generator(kind) + capability = next(self._capability_generators[kind]) return capability - @inlineCallbacks - def add_data(self, key, data): + def add_data(self, kind, data): """ - adds some data to our grid, returning a capability + adds some data to our grid + + :returns: a capability-string """ - cap = yield create_fake_capability("URI:CHK:", key, data) + assert isinstance(data, bytes) + + cap = self._generate_capability(kind) if self._data is None: self._data = dict() + assert cap not in self._data, "already have '{}'".format(cap) self._data[cap] = data - returnValue(cap) + return cap def render_PUT(self, request): data = request.content.read() -# if len(data) == 0: -# raise RuntimeError("No empty data") - - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap + return self.add_data("URI:CHK:", data) def render_POST(self, request): - kind = request.args[u"t"][0] + t = request.args[u"t"][0] data = request.content.read() - cap = self._generate_cap() - if self._data is None: - self._data = dict() - self._data[cap] = data - return cap -# if kind == 'mkdir-immutable': + type_to_kind = { + "mkdir-immutable": "URI:DIR2-CHK:" + } + kind = type_to_kind[t] + return self.add_data(kind, data) def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -309,17 +177,15 @@ class _FakeTahoeUriHandler(Resource): return self._data[capability] -@inlineCallbacks def create_fake_tahoe_root(): """ - Probably should take some params to control what this fake does: - return errors, pre-populate capabilities, ... + :returns: an IResource instance that will handle certain Tahoe URI + endpoints similar to a real Tahoe server. """ root = _FakeTahoeRoot( uri=_FakeTahoeUriHandler(), ) - yield - returnValue(root) + return root @implementer(IBodyProducer) @@ -356,16 +222,15 @@ class _SynchronousProducer(object): return succeed(None) -@inlineCallbacks def create_tahoe_treq_client(root=None): """ """ if root is None: - root = yield create_fake_tahoe_root() + root = create_fake_tahoe_root() client = HTTPClient( agent=RequestTraversalAgent(root), data_to_body_producer=_SynchronousProducer, ) - returnValue(client) + return client From 213d0348a890fc1ca34ad7c24e8c2478a0f03465 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 13:36:28 -0600 Subject: [PATCH 071/134] news --- newsfragments/3317.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3317.feature diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature new file mode 100644 index 000000000..5ed00acf5 --- /dev/null +++ b/newsfragments/3317.feature @@ -0,0 +1 @@ +Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file From 86a90084bb2082c51b783a207d2a93c4a900393c Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 14:14:42 -0600 Subject: [PATCH 072/134] simple test of testing/web --- src/allmydata/test/test_testing.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/allmydata/test/test_testing.py diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py new file mode 100644 index 000000000..37e68a0de --- /dev/null +++ b/src/allmydata/test/test_testing.py @@ -0,0 +1,40 @@ +import io + +from twisted.trial.unittest import ( + TestCase, +) +from twisted.internet.defer import ( + inlineCallbacks, +) + +from allmydata.testing.web import ( + create_tahoe_treq_client, +) + + +class FakeWebTest(TestCase): + """ + Test the WebUI verified-fakes infrastucture + """ + + def setUp(self): + super(FakeWebTest, self).setUp() + self.http_client = create_tahoe_treq_client() + + @inlineCallbacks + def test_create_and_download(self): + """ + Upload some content and then download it + """ + content = "fake data\n" * 100 + resp = yield self.http_client.put("http://example.com/uri", content) + cap = yield resp.content() + + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + round_trip_content = yield resp.content() + + self.assertEqual(content, round_trip_content) From 00e7d01a1af8c1dec324305c976e76010cc2c73d Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:40 -0600 Subject: [PATCH 073/134] unused imports --- src/allmydata/test/test_testing.py | 2 -- src/allmydata/testing/web.py | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37e68a0de..da37b1de3 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,5 +1,3 @@ -import io - from twisted.trial.unittest import ( TestCase, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 78d2b7931..bc0138326 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,30 +1,17 @@ -import io -import os -import json -import string import hashlib import attr from hyperlink import DecodedURL -from twisted.python.filepath import ( - FilePath, -) from twisted.web.resource import ( Resource, ) -from twisted.web.client import ( - Agent, - FileBodyProducer, -) from twisted.web.iweb import ( IBodyProducer, ) from twisted.internet.defer import ( - inlineCallbacks, succeed, - returnValue, ) from treq.client import ( @@ -32,8 +19,6 @@ from treq.client import ( ) from treq.testing import ( RequestTraversalAgent, - RequestSequence, - StubTreq, ) from zope.interface import implementer From 36d9bf5d0983f66667cf1c9910736d5644e0a1af Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:34:47 -0600 Subject: [PATCH 074/134] new-style classes only --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index bc0138326..de515ebc9 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -28,7 +28,7 @@ from allmydata.util import ( ) -class _FakeTahoeRoot(Resource): +class _FakeTahoeRoot(Resource, object): """ This is a sketch of how an in-memory 'fake' of a Tahoe WebUI. Ultimately, this will live in Tahoe @@ -100,7 +100,7 @@ def capability_generator(kind): yield cap.encode("ascii") -class _FakeTahoeUriHandler(Resource): +class _FakeTahoeUriHandler(Resource, object): """ """ From 0f4201ef193b9af183e06e6141ba659a240de4b0 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 15:55:19 -0600 Subject: [PATCH 075/134] the tests are unit-tests --- integration/test_testing.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 integration/test_testing.py diff --git a/integration/test_testing.py b/integration/test_testing.py deleted file mode 100644 index 47498b1d7..000000000 --- a/integration/test_testing.py +++ /dev/null @@ -1,26 +0,0 @@ - - -from allmydata.testing.web import ( - create_fake_tahoe_root, - deterministic_key_generator, -) - -import pytest -import pytest_twisted - - -@pytest_twisted.inlineCallbacks -def test_retrieve_cap(): - """ - WebUI Fake can serve a read-capability back - """ - - keys = deterministic_key_generator() - root = yield create_fake_tahoe_root() - dummy_readcap = yield root.add_data( - next(keys), - "some dummy content\n"*20 - ) - print("readcap: {}".format(dummy_readcap)) - - assert dummy_readcap.to_string() == "URI:CHK:ifaucqkbifaucqkbifaucqkbie:qg3n5r4q36wvwt33fobghsyqacc5ymnadk7wf6hafamsh6amybza:1:1:380" From 0e5422c24b82f90b7fcb3cf46d2076aec4a31ff9 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:11 -0600 Subject: [PATCH 076/134] handle more inputs in test body-producer --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index de515ebc9..aa3b2a4a7 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -16,6 +16,7 @@ from twisted.internet.defer import ( from treq.client import ( HTTPClient, + FileBodyProducer, ) from treq.testing import ( RequestTraversalAgent, @@ -192,9 +193,12 @@ class _SynchronousProducer(object): """ Create a synchronous producer with some bytes. """ + if isinstance(body, FileBodyProducer): + body = body._inputFile.read() + if not isinstance(body, bytes): raise ValueError( - "'body' must be bytes" + "'body' must be bytes not '{}'".format(type(body)) ) self.body = body self.length = len(body) From 318dc5092d141e2c2e32f9b18a65aeba78ca65bf Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 11 Jun 2020 19:57:21 -0600 Subject: [PATCH 077/134] key -> kind --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aa3b2a4a7..fa6a2555a 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -40,8 +40,8 @@ class _FakeTahoeRoot(Resource, object): self._uri = uri self.putChild(b"uri", self._uri) - def add_data(self, key, data): - return self._uri.add_data(key, data) + def add_data(self, kind, data): + return self._uri.add_data(kind, data) @attr.s From 05ea0a2fe9d8525d85c6e6924ce4db35cf479462 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:03 -0600 Subject: [PATCH 078/134] copyright, header --- src/allmydata/test/test_testing.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index da37b1de3..4dc9c6bb6 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -1,6 +1,16 @@ -from twisted.trial.unittest import ( - TestCase, -) +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Tests for the allmydata.testing helpers +""" + from twisted.internet.defer import ( inlineCallbacks, ) From 776ee5f91c12ab3e1577c947f5ed92bf4e414a23 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:04:12 -0600 Subject: [PATCH 079/134] use hypothesis -- which doesn't know about Deferred, so need testtools too --- src/allmydata/test/test_testing.py | 52 +++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 4dc9c6bb6..e350546ca 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -19,6 +19,23 @@ from allmydata.testing.web import ( create_tahoe_treq_client, ) +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + binary, +) + +from testtools import ( + TestCase, +) +from testtools.matchers import ( + Always, +) +from testtools.twistedsupport import ( + succeeded, +) + class FakeWebTest(TestCase): """ @@ -29,20 +46,31 @@ class FakeWebTest(TestCase): super(FakeWebTest, self).setUp() self.http_client = create_tahoe_treq_client() - @inlineCallbacks - def test_create_and_download(self): + @given( + content=binary(), + ) + def test_create_and_download(self, content): """ - Upload some content and then download it + Upload some content (via 'PUT /uri') and then download it (via + 'GET /uri?uri=...') """ - content = "fake data\n" * 100 - resp = yield self.http_client.put("http://example.com/uri", content) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + @inlineCallbacks + def do_test(): + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 200) - resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + cap = yield resp.content() + self.assertTrue(cap.startswith("URI:CHK:")) + + resp = yield self.http_client.get( + "http://example.com/uri?uri={}".format(cap) + ) + self.assertEqual(resp.code, 200) + + round_trip_content = yield resp.content() + self.assertEqual(content, round_trip_content) + self.assertThat( + do_test(), + succeeded(Always()), ) - round_trip_content = yield resp.content() - - self.assertEqual(content, round_trip_content) From 949b09ce278ef65689d853b1bf3b33caa330a02c Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:08:36 -0600 Subject: [PATCH 080/134] actually returns CREATED for new data --- src/allmydata/test/test_testing.py | 2 +- src/allmydata/testing/web.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index e350546ca..a5c712d77 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -58,7 +58,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 200) + self.assertEqual(resp.code, 201) cap = yield resp.content() self.assertTrue(cap.startswith("URI:CHK:")) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index fa6a2555a..99dabeddf 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -140,6 +140,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() + request.setResponseCode(201) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From fa597e1735fe64b6df8288ee75f7136b713b58b6 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 22:57:38 -0600 Subject: [PATCH 081/134] use real parser --- src/allmydata/test/test_testing.py | 11 ++++++++--- src/allmydata/testing/web.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a5c712d77..b5bbacad4 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -15,6 +15,10 @@ from twisted.internet.defer import ( inlineCallbacks, ) +from allmydata.uri import ( + from_string, + CHKFileURI, +) from allmydata.testing.web import ( create_tahoe_treq_client, ) @@ -60,11 +64,12 @@ class FakeWebTest(TestCase): resp = yield self.http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) - cap = yield resp.content() - self.assertTrue(cap.startswith("URI:CHK:")) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) resp = yield self.http_client.get( - "http://example.com/uri?uri={}".format(cap) + "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 99dabeddf..f6411792d 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -90,7 +90,7 @@ def capability_generator(kind): key = base32.b2a(key_hasher.digest()[:16]) # key is 16 bytes ueb_hash = base32.b2a(ueb_hasher.digest()) # ueb hash is 32 bytes - cap = u"{kind}:{key}:{ueb_hash}:{n}:{k}:{size}".format( + cap = u"{kind}{key}:{ueb_hash}:{n}:{k}:{size}".format( kind=kind, key=key, ueb_hash=ueb_hash, From e79534abe8a45e4a5d284e40833310bdda13af71 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:01:02 -0600 Subject: [PATCH 082/134] housekeeping --- src/allmydata/testing/web.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index f6411792d..2b0f830f2 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -1,3 +1,16 @@ +# -*- coding: utf-8 -*- +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2020 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +""" +Test-helpers for clients that use the WebUI. +""" + import hashlib import attr @@ -28,6 +41,11 @@ from allmydata.util import ( base32, ) +__all__ = ( + "create_fake_tahoe_root", + "create_tahoe_treq_client", +) + class _FakeTahoeRoot(Resource, object): """ From 23b17ca02e0bc2e59c3f5982e454bc28f19c77ec Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:05:32 -0600 Subject: [PATCH 083/134] cleanup, docstrings --- src/allmydata/testing/web.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 2b0f830f2..7d9b5730b 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -49,8 +49,8 @@ __all__ = ( class _FakeTahoeRoot(Resource, object): """ - This is a sketch of how an in-memory 'fake' of a Tahoe - WebUI. Ultimately, this will live in Tahoe + An in-memory 'fake' of a Tahoe WebUI root. Currently it only + implements (some of) the `/uri` resource. """ def __init__(self, uri=None): @@ -62,13 +62,6 @@ class _FakeTahoeRoot(Resource, object): return self._uri.add_data(kind, data) -@attr.s -class _FakeCapability(object): - """ - """ - data=attr.ib() - - KNOWN_CAPABILITIES = [ getattr(allmydata.uri, t).BASE_STRING for t in dir(allmydata.uri) @@ -121,6 +114,8 @@ def capability_generator(kind): class _FakeTahoeUriHandler(Resource, object): """ + An in-memory fake of (some of) the `/uri` endpoint of a Tahoe + WebUI """ isLeaf = True @@ -232,6 +227,12 @@ class _SynchronousProducer(object): def create_tahoe_treq_client(root=None): """ + :param root: an instance created via `create_fake_tahoe_root`. The + caller might want a copy of this to call `.add_data` for example. + + :returns: an instance of treq.client.HTTPClient wired up to + in-memory fakes of the Tahoe WebUI. Only a subset of the real + WebUI is available. """ if root is None: From c64e78634d7402df1348cc2ed02ec1559eb851d6 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:06:33 -0600 Subject: [PATCH 084/134] docstring --- src/allmydata/testing/web.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7d9b5730b..64e615cd6 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -54,6 +54,9 @@ class _FakeTahoeRoot(Resource, object): """ def __init__(self, uri=None): + """ + :param uri: a Resource to handle the `/uri` tree. + """ Resource.__init__(self) # this is an old-style class :( self._uri = uri self.putChild(b"uri", self._uri) From a38188164f7ac7d60b20ff28d029b1c48faea4da Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 12 Jun 2020 23:08:04 -0600 Subject: [PATCH 085/134] comment --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 64e615cd6..c2bced12e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -93,7 +93,7 @@ def capability_generator(kind): # different sequences. number = 0 key_hasher = hashlib.new("sha256") - ueb_hasher = hashlib.new("sha256") + ueb_hasher = hashlib.new("sha256") # ueb means "URI Extension Block" # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size" while True: From 9f6fcde2f1f0bdfe59eb234960b63913dfaac867 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:12 -0600 Subject: [PATCH 086/134] more docstring --- src/allmydata/testing/web.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index c2bced12e..63f6b9877 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -74,6 +74,10 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ + Deterministically generates a streap of valid capabilities of the + given kind. The N, K and size values aren't related to anything + real. + :param str kind: the kind of capability, like `URI:CHK` :returns: a generator that yields new capablities of a particular From 76040b53f5afd2c7099250ba7e825718f1c1fdf0 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:39:57 -0600 Subject: [PATCH 087/134] assert -> raise --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 63f6b9877..90b50d2e0 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -149,7 +149,8 @@ class _FakeTahoeUriHandler(Resource, object): :returns: a capability-string """ - assert isinstance(data, bytes) + if not isinstance(data, bytes): + raise TypeError("'data' must be bytes") cap = self._generate_capability(kind) if self._data is None: From 24899dae8e803d4686c5a534a0784057900ea036 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:41:22 -0600 Subject: [PATCH 088/134] assert -> ValueError --- src/allmydata/testing/web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 90b50d2e0..5e6d564fa 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -155,7 +155,8 @@ class _FakeTahoeUriHandler(Resource, object): cap = self._generate_capability(kind) if self._data is None: self._data = dict() - assert cap not in self._data, "already have '{}'".format(cap) + if cap in self._data: + raise ValueError("already have '{}'".format(cap)) self._data[cap] = data return cap From b952ef066c556e5d17d321328abe4222f94523ae Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 00:46:55 -0600 Subject: [PATCH 089/134] match query args --- src/allmydata/testing/web.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5e6d564fa..3231c5820 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -177,8 +177,16 @@ class _FakeTahoeUriHandler(Resource, object): def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) - # XXX FIXME - capability = uri.query[0][1] + capability = None + for arg, value in uri.query: + if arg == u"uri": + capability = value + if capability is None: + raise Exception( + "No ?uri= arguent in GET '{}'".format( + uri.to_string() + ) + ) if self._data is None or capability not in self._data: return u"No data for '{}'".format(capability).decode("ascii") From 0cbdbd9ebef8834d195ea43871d9c2a03e789ece Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:14:45 -0600 Subject: [PATCH 090/134] symbols, not magic-numbers --- src/allmydata/testing/web.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 3231c5820..5ab870f4f 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -23,6 +23,10 @@ from twisted.web.resource import ( from twisted.web.iweb import ( IBodyProducer, ) +from twisted.web import ( + http, +) + from twisted.internet.defer import ( succeed, ) @@ -162,7 +166,7 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() - request.setResponseCode(201) # real code does this for brand-new files + request.setResponseCode(http.CREATED) # real code does this for brand-new files return self.add_data("URI:CHK:", data) def render_POST(self, request): From 02a543515f459e2fb8ab92a2b8d96bd21ae216dc Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 01:55:14 -0600 Subject: [PATCH 091/134] replciate Tahoe's behavior for replace=true/false --- src/allmydata/test/test_testing.py | 21 +++++++++++++++++++- src/allmydata/testing/web.py | 31 +++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index b5bbacad4..a995e756d 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -61,7 +61,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri", content) + resp = yield self.http_client.put("http://example.com/uri?replace=true", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() @@ -79,3 +79,22 @@ class FakeWebTest(TestCase): do_test(), succeeded(Always()), ) + + @inlineCallbacks + def test_duplicate_upload(self): + """ + Upload the same content (via 'PUT /uri') twice with no overwrite + """ + + content = "fake content\n" * 200 + + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) + + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) + + # this one fails: same content but no ?replace=true + resp = yield self.http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 409) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5ab870f4f..ef7cd5bed 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,6 +30,9 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) +from twisted.python.failure import ( + Failure, +) from treq.client import ( HTTPClient, @@ -44,6 +47,13 @@ import allmydata.uri from allmydata.util import ( base32, ) +from allmydata.interfaces import ( + ExistingChildError, +) +from allmydata.web.common import ( + humanize_failure, +) + __all__ = ( "create_fake_tahoe_root", @@ -147,7 +157,7 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self._capability_generators[kind]) return capability - def add_data(self, kind, data): + def add_data(self, kind, data, allow_duplicate=False): """ adds some data to our grid @@ -156,9 +166,18 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - cap = self._generate_capability(kind) if self._data is None: self._data = dict() + + for k in self._data: + if self._data[k] == data: + if allow_duplicate: + return k + raise ValueError( + "Duplicate data" + ) + + cap = self._generate_capability(kind) if cap in self._data: raise ValueError("already have '{}'".format(cap)) self._data[cap] = data @@ -167,7 +186,13 @@ class _FakeTahoeUriHandler(Resource, object): def render_PUT(self, request): data = request.content.read() request.setResponseCode(http.CREATED) # real code does this for brand-new files - return self.add_data("URI:CHK:", data) + replace = request.args.get("replace", None) + try: + return self.add_data("URI:CHK:", data, allow_duplicate=replace) + except ValueError: + msg, code = humanize_failure(Failure(ExistingChildError())) + request.setResponseCode(code) + return msg def render_POST(self, request): t = request.args[u"t"][0] From d221787541940dbe0e70f24cd8def42188666e8f Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:03:19 -0600 Subject: [PATCH 092/134] better docstring --- src/allmydata/testing/web.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ef7cd5bed..5f011f8b1 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -225,6 +225,14 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ + If you wish to pre-populate data into the fake Tahoe grid, retain + a reference to this root and pass it to + `create_tahoe_treq_client`. For example:: + + root = create_fake_tahoe_root() + root.add_data(...) + client = create_tahoe_treq_client(root) + :returns: an IResource instance that will handle certain Tahoe URI endpoints similar to a real Tahoe server. """ From caf77be8ae5b858cf4c7eaf54568baeec29db9fd Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 02:05:27 -0600 Subject: [PATCH 093/134] different words for news --- newsfragments/3317.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3317.feature b/newsfragments/3317.feature index 5ed00acf5..2a7048397 100644 --- a/newsfragments/3317.feature +++ b/newsfragments/3317.feature @@ -1 +1 @@ -Verified fakes for (some of) Tahoe's WebUI via treq \ No newline at end of file +allmydata.testing.web, a new module, now offers a supported Python API for testing Tahoe-LAFS web API clients. \ No newline at end of file From 939f60f6207b6bf13e42fa37893a181ed6168288 Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:22:17 -0600 Subject: [PATCH 094/134] embetter docstring --- src/allmydata/testing/web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 5f011f8b1..ca010afa4 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -226,11 +226,11 @@ class _FakeTahoeUriHandler(Resource, object): def create_fake_tahoe_root(): """ If you wish to pre-populate data into the fake Tahoe grid, retain - a reference to this root and pass it to + a reference to this root by creating it yourself and passing it to `create_tahoe_treq_client`. For example:: root = create_fake_tahoe_root() - root.add_data(...) + cap_string = root.add_data(...) client = create_tahoe_treq_client(root) :returns: an IResource instance that will handle certain Tahoe URI From 6a37f32eeff5020e3a96146433f602c0e532a3be Mon Sep 17 00:00:00 2001 From: meejah Date: Sat, 13 Jun 2020 17:28:28 -0600 Subject: [PATCH 095/134] use attr.s --- src/allmydata/testing/web.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index ca010afa4..7c6fc1996 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -133,6 +133,7 @@ def capability_generator(kind): yield cap.encode("ascii") +@attr.s class _FakeTahoeUriHandler(Resource, object): """ An in-memory fake of (some of) the `/uri` endpoint of a Tahoe @@ -140,8 +141,9 @@ class _FakeTahoeUriHandler(Resource, object): """ isLeaf = True - _data = None - _capability_generators = None + + data = attr.ib(default=attr.Factory(dict)) + capability_generators = attr.ib(default=attr.Factory(dict)) def _generate_capability(self, kind): """ @@ -149,12 +151,9 @@ class _FakeTahoeUriHandler(Resource, object): :returns: the next capability-string for the given kind """ - if self._capability_generators is None: - self._capability_generators = dict() - - if kind not in self._capability_generators: - self._capability_generators[kind] = capability_generator(kind) - capability = next(self._capability_generators[kind]) + if kind not in self.capability_generators: + self.capability_generators[kind] = capability_generator(kind) + capability = next(self.capability_generators[kind]) return capability def add_data(self, kind, data, allow_duplicate=False): @@ -166,11 +165,8 @@ class _FakeTahoeUriHandler(Resource, object): if not isinstance(data, bytes): raise TypeError("'data' must be bytes") - if self._data is None: - self._data = dict() - - for k in self._data: - if self._data[k] == data: + for k in self.data: + if self.data[k] == data: if allow_duplicate: return k raise ValueError( @@ -178,9 +174,9 @@ class _FakeTahoeUriHandler(Resource, object): ) cap = self._generate_capability(kind) - if cap in self._data: + if cap in self.data: raise ValueError("already have '{}'".format(cap)) - self._data[cap] = data + self.data[cap] = data return cap def render_PUT(self, request): @@ -217,10 +213,10 @@ class _FakeTahoeUriHandler(Resource, object): ) ) - if self._data is None or capability not in self._data: + if capability not in self.data: return u"No data for '{}'".format(capability).decode("ascii") - return self._data[capability] + return self.data[capability] def create_fake_tahoe_root(): From bd20f8d6f4ea89b178ab8389af1a86cb0288b4c7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 16 Jun 2020 12:10:44 -0400 Subject: [PATCH 096/134] Update newsfragment text --- newsfragments/3312.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3312.bugfix b/newsfragments/3312.bugfix index ed00922a6..9939fe1f0 100644 --- a/newsfragments/3312.bugfix +++ b/newsfragments/3312.bugfix @@ -1 +1 @@ -Add your info here \ No newline at end of file +Make directory page links work. From ef71fc2bfc743548e47a76578a88a18d9bd7ed26 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:15:01 -0600 Subject: [PATCH 097/134] Get rid of ?replace= handling entirely and more-correctly support /uri variants --- src/allmydata/test/test_testing.py | 98 ++++++++++++++++++++++++------ src/allmydata/testing/web.py | 52 ++++++++-------- 2 files changed, 107 insertions(+), 43 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index a995e756d..67c38ed57 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -21,6 +21,11 @@ from allmydata.uri import ( ) from allmydata.testing.web import ( create_tahoe_treq_client, + capability_generator, +) + +from hyperlink import ( + DecodedURL, ) from hypothesis import ( @@ -35,9 +40,12 @@ from testtools import ( ) from testtools.matchers import ( Always, + Equals, + MatchesStructure, ) from testtools.twistedsupport import ( succeeded, + failed, ) @@ -46,9 +54,9 @@ class FakeWebTest(TestCase): Test the WebUI verified-fakes infrastucture """ - def setUp(self): - super(FakeWebTest, self).setUp() - self.http_client = create_tahoe_treq_client() + # Note: do NOT use setUp() because Hypothesis doesn't work + # properly with it. You must instead do all fixture-type work + # yourself in each test. @given( content=binary(), @@ -58,21 +66,31 @@ class FakeWebTest(TestCase): Upload some content (via 'PUT /uri') and then download it (via 'GET /uri?uri=...') """ + http_client = create_tahoe_treq_client() @inlineCallbacks def do_test(): - resp = yield self.http_client.put("http://example.com/uri?replace=true", content) + resp = yield http_client.put("http://example.com/uri", content) self.assertEqual(resp.code, 201) cap_raw = yield resp.content() cap = from_string(cap_raw) self.assertIsInstance(cap, CHKFileURI) - resp = yield self.http_client.get( + resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() + + # using the form "/uri/" is also valid + + resp = yield http_client.get( + "http://example.com/uri/{}".format(cap.to_string()) + ) + self.assertEqual(resp.code, 200) + round_trip_content = yield resp.content() self.assertEqual(content, round_trip_content) self.assertThat( @@ -80,21 +98,67 @@ class FakeWebTest(TestCase): succeeded(Always()), ) - @inlineCallbacks - def test_duplicate_upload(self): + @given( + content=binary(), + ) + def test_duplicate_upload(self, content): """ - Upload the same content (via 'PUT /uri') twice with no overwrite + Upload the same content (via 'PUT /uri') twice """ - content = "fake content\n" * 200 + http_client = create_tahoe_treq_client() - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + @inlineCallbacks + def do_test(): + resp = yield http_client.put("http://example.com/uri", content) + self.assertEqual(resp.code, 201) - cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + cap_raw = yield resp.content() + cap = from_string(cap_raw) + self.assertIsInstance(cap, CHKFileURI) - # this one fails: same content but no ?replace=true - resp = yield self.http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 409) + resp = yield http_client.put("http://example.com/uri", content) + self.assertThat(resp.code, Equals(200)) + self.assertThat( + do_test(), + succeeded(Always()), + ) + + def test_download_missing(self): + """ + Error if we download a capability that doesn't exist + """ + + http_client = create_tahoe_treq_client() + cap_gen = capability_generator("URI:CHK:") + + uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(next(cap_gen))) + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(500) + ) + ) + ) + + def test_download_no_arg(self): + """ + Error if we GET from "/uri" with no ?uri= query-arg + """ + + http_client = create_tahoe_treq_client() + + uri = DecodedURL.from_text(u"http://example.com/uri/") + resp = http_client.get(uri.to_uri().to_text()) + + self.assertThat( + resp, + succeeded( + MatchesStructure( + code=Equals(400) + ) + ) + ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 7c6fc1996..aaa795d06 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -76,7 +76,8 @@ class _FakeTahoeRoot(Resource, object): self.putChild(b"uri", self._uri) def add_data(self, kind, data): - return self._uri.add_data(kind, data) + fresh, cap = self._uri.add_data(kind, data) + return cap KNOWN_CAPABILITIES = [ @@ -156,39 +157,32 @@ class _FakeTahoeUriHandler(Resource, object): capability = next(self.capability_generators[kind]) return capability - def add_data(self, kind, data, allow_duplicate=False): + def add_data(self, kind, data): """ adds some data to our grid - :returns: a capability-string + :returns: a two-tuple: a bool (True if the data is freshly added) and a capability-string """ if not isinstance(data, bytes): raise TypeError("'data' must be bytes") for k in self.data: if self.data[k] == data: - if allow_duplicate: - return k - raise ValueError( - "Duplicate data" - ) + return (False, k) cap = self._generate_capability(kind) - if cap in self.data: - raise ValueError("already have '{}'".format(cap)) + assert cap not in self.data, "logic error" # shouldn't happen self.data[cap] = data - return cap + return (True, cap) def render_PUT(self, request): data = request.content.read() - request.setResponseCode(http.CREATED) # real code does this for brand-new files - replace = request.args.get("replace", None) - try: - return self.add_data("URI:CHK:", data, allow_duplicate=replace) - except ValueError: - msg, code = humanize_failure(Failure(ExistingChildError())) - request.setResponseCode(code) - return msg + fresh, cap = self.add_data("URI:CHK:", data) + if fresh: + request.setResponseCode(http.CREATED) # real code does this for brand-new files + else: + request.setResponseCode(http.OK) # replaced/modified files + return cap def render_POST(self, request): t = request.args[u"t"][0] @@ -198,7 +192,8 @@ class _FakeTahoeUriHandler(Resource, object): "mkdir-immutable": "URI:DIR2-CHK:" } kind = type_to_kind[t] - return self.add_data(kind, data) + fresh, cap = self.add_data(kind, data) + return cap def render_GET(self, request): uri = DecodedURL.from_text(request.uri.decode('utf8')) @@ -206,14 +201,19 @@ class _FakeTahoeUriHandler(Resource, object): for arg, value in uri.query: if arg == u"uri": capability = value - if capability is None: - raise Exception( - "No ?uri= arguent in GET '{}'".format( - uri.to_string() - ) - ) + # it's legal to use the form "/uri/" + if capability is None and request.postpath and request.postpath[0]: + capability = request.postpath[0] + # if we don't yet have a capability, that's an error + if capability is None: + request.setResponseCode(http.BAD_REQUEST) + return b"GET /uri requires uri=" + + # the user gave us a capability; if our Grid doesn't have any + # data for it, that's an error. if capability not in self.data: + request.setResponseCode(http.BAD_REQUEST) return u"No data for '{}'".format(capability).decode("ascii") return self.data[capability] From 57eb875354f42b6bd037bb8f3ec0fda25a6ce333 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:16:39 -0600 Subject: [PATCH 098/134] testtool's asserts --- src/allmydata/test/test_testing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 67c38ed57..6ba622f3c 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -71,7 +71,7 @@ class FakeWebTest(TestCase): @inlineCallbacks def do_test(): resp = yield http_client.put("http://example.com/uri", content) - self.assertEqual(resp.code, 201) + self.assertThat(resp.code, Equals(201)) cap_raw = yield resp.content() cap = from_string(cap_raw) @@ -80,7 +80,7 @@ class FakeWebTest(TestCase): resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) ) - self.assertEqual(resp.code, 200) + self.assertThat(resp.code, Equals(200)) round_trip_content = yield resp.content() From f0ac5a2cef89080c7fe77f73fd63e27f6bc6d232 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:19:30 -0600 Subject: [PATCH 099/134] more-testtools-y asserts --- src/allmydata/test/test_testing.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 6ba622f3c..37a5f3c7b 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -41,7 +41,9 @@ from testtools import ( from testtools.matchers import ( Always, Equals, + IsInstance, MatchesStructure, + AfterPreprocessing, ) from testtools.twistedsupport import ( succeeded, @@ -75,7 +77,7 @@ class FakeWebTest(TestCase): cap_raw = yield resp.content() cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat(cap, IsInstance(CHKFileURI)) resp = yield http_client.get( "http://example.com/uri?uri={}".format(cap.to_string()) @@ -114,8 +116,13 @@ class FakeWebTest(TestCase): self.assertEqual(resp.code, 201) cap_raw = yield resp.content() - cap = from_string(cap_raw) - self.assertIsInstance(cap, CHKFileURI) + self.assertThat( + cap_raw, + AfterPreprocessing( + from_string, + IsInstance(CHKFileURI) + ) + ) resp = yield http_client.put("http://example.com/uri", content) self.assertThat(resp.code, Equals(200)) From aedc58ef5158f43b6809b1afeab207ad7c9ba2af Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 19 Jun 2020 17:22:42 -0600 Subject: [PATCH 100/134] unused imports --- src/allmydata/test/test_testing.py | 1 - src/allmydata/testing/web.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py index 37a5f3c7b..ec6935914 100644 --- a/src/allmydata/test/test_testing.py +++ b/src/allmydata/test/test_testing.py @@ -47,7 +47,6 @@ from testtools.matchers import ( ) from testtools.twistedsupport import ( succeeded, - failed, ) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index aaa795d06..6656236ca 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -30,9 +30,6 @@ from twisted.web import ( from twisted.internet.defer import ( succeed, ) -from twisted.python.failure import ( - Failure, -) from treq.client import ( HTTPClient, @@ -47,12 +44,6 @@ import allmydata.uri from allmydata.util import ( base32, ) -from allmydata.interfaces import ( - ExistingChildError, -) -from allmydata.web.common import ( - humanize_failure, -) __all__ = ( From 6ab7f26faf4899caee412279c3812fd4ba6de486 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:28:31 -0400 Subject: [PATCH 101/134] Make UploadStatusPage a Resource --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 8147437fa..ee0e924e0 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -183,7 +183,7 @@ class UploadResultsRendererMixin(Element): return d -class UploadStatusPage(MultiFormatResource): +class UploadStatusPage(Resource, object): """Renders /status/up-%d.""" def __init__(self, upload_status): @@ -193,7 +193,7 @@ class UploadStatusPage(MultiFormatResource): super(UploadStatusPage, self).__init__() self._upload_status = upload_status - def render_HTML(self, req): + def render_GET(self, req): elem = UploadStatusElement(self._upload_status) return renderElement(req, elem) From dfbb8e7743f07cb5369f64c8949c1ac99bab5036 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:53:39 -0400 Subject: [PATCH 102/134] Make UploadResultsPage a Resource --- src/allmydata/web/unlinked.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 91efff2c4..09795101a 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -3,6 +3,7 @@ import urllib from twisted.web import http from twisted.internet import defer from twisted.python.filepath import FilePath +from twisted.web.resource import Resource from twisted.web.template import ( XMLFile, renderer, @@ -19,7 +20,6 @@ from allmydata.web.common import ( WebError, get_format, get_mutable_type, - MultiFormatResource, ) from allmydata.web import status @@ -73,7 +73,7 @@ def POSTUnlinkedCHK(req, client): return d -class UploadResultsPage(MultiFormatResource): +class UploadResultsPage(Resource, object): """'POST /uri', to create an unlinked file.""" def __init__(self, upload_results): @@ -83,7 +83,7 @@ class UploadResultsPage(MultiFormatResource): super(UploadResultsPage, self).__init__() self._upload_results = upload_results - def render_HTML(self, req): + def render_POST(self, req): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) From 88d3fa7415505c3c7a305d1613a9faf7e3fc111d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 23 Jun 2020 21:54:23 -0400 Subject: [PATCH 103/134] Remove now-unnecessary UploadResultsPage.render_UPLOAD --- src/allmydata/web/unlinked.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 09795101a..df342d7dd 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -87,23 +87,6 @@ class UploadResultsPage(Resource, object): elem = UploadResultsElement(self._upload_results) return renderElement(req, elem) - # This is weird but necessary because: - # - # 1. MultiFormatResource.render() uses argument "t" to figure out - # its output format. - # - # 2. Upload request is of the form "POST /uri?t=upload&file=newfile". - # See URIHandler.render_POST(). - # - # MultiFormatResource.render() looks up "t" argument, which in - # this case has the value "upload", and then it would look for a - # render_UPLOAD() method. - # - # We could change upload request to use more descriptive names - # that do not cause name collisions like this. That should be a - # separate change though. - render_UPLOAD = render_HTML - class UploadResultsElement(status.UploadResultsRendererMixin): From a9f4210014df81b1a7d5b914f758aecd64e34b54 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:15:58 -0600 Subject: [PATCH 104/134] streap -> stream --- src/allmydata/testing/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 6656236ca..0d6e7cb6e 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -80,7 +80,7 @@ KNOWN_CAPABILITIES = [ def capability_generator(kind): """ - Deterministically generates a streap of valid capabilities of the + Deterministically generates a stream of valid capabilities of the given kind. The N, K and size values aren't related to anything real. From 80359e02e6e255d48ade2060e413447b2728a63e Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 30 Jun 2020 08:17:34 -0600 Subject: [PATCH 105/134] assert -> Exception --- src/allmydata/testing/web.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py index 0d6e7cb6e..12f5ef40c 100644 --- a/src/allmydata/testing/web.py +++ b/src/allmydata/testing/web.py @@ -162,7 +162,10 @@ class _FakeTahoeUriHandler(Resource, object): return (False, k) cap = self._generate_capability(kind) - assert cap not in self.data, "logic error" # shouldn't happen + # it should be impossible for this to already be in our data, + # but check anyway to be sure + if cap in self.data: + raise Exception("Internal error; key already exists somehow") self.data[cap] = data return (True, cap) From 9a2da37241305472484f3ae1a6c6698fbe45cd49 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:41:17 -0400 Subject: [PATCH 106/134] Here's a start --- docs/CODE_OF_CONDUCT.md | 53 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 54 insertions(+) create mode 100644 docs/CODE_OF_CONDUCT.md diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..868f59c6a --- /dev/null +++ b/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,53 @@ +# Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of +fostering an open and welcoming community, we pledge to respect all people who +contribute through reporting issues, posting feature requests, updating +documentation, submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free +experience for everyone, regardless of level of experience, gender, gender +identity and expression, sexual orientation, disability, personal appearance, +body size, race, ethnicity, age, religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic + addresses, without explicit permission +* Other unethical or unprofessional conduct + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +By adopting this Code of Conduct, project maintainers commit themselves to +fairly and consistently applying these principles to every aspect of managing +this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting a project maintainer (see below). All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. Maintainers are +obligated to maintain confidentiality with regard to the reporter of an +incident. + +The following community members have made themselves available for conduct issues: + +- Jean-Paul Calderone (jean-paul at leastauthority dot com) + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.3.0, available at +[http://contributor-covenant.org/version/1/3/0/][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/3/0/ diff --git a/docs/index.rst b/docs/index.rst index 98b4f241b..b125cdd4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,6 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING + ../.github/CODE_OF_CONDUCT servers helper From f9cab479fd55dca3d8e1e4239f5503d96898ad7e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:48:39 -0400 Subject: [PATCH 107/134] as far as I can tell there is nothing magic about putting it in .github --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index b125cdd4e..581e74bbb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Contents: known_issues ../.github/CONTRIBUTING - ../.github/CODE_OF_CONDUCT + CODE_OF_CONDUCT servers helper From f9ac0a7a9eb178dc998a2bbb76f3e8efb3fcf212 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 1 Jul 2020 15:50:47 -0400 Subject: [PATCH 108/134] news fragment --- newsfragments/2755.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2755.other diff --git a/newsfragments/2755.other b/newsfragments/2755.other new file mode 100644 index 000000000..e3e31465d --- /dev/null +++ b/newsfragments/2755.other @@ -0,0 +1 @@ +The Tahoe-LAFS project has adopted a formal code of conduct. From b7f88b278f3c157213577faf23f397a7c3709b83 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 13:40:40 -0400 Subject: [PATCH 109/134] Split out tests for humanreadable into their own module, so we can port just humanreadable.py in isolation. --- src/allmydata/test/test_humanreadable.py | 46 ++++++++++++++++++++++++ src/allmydata/test/test_util.py | 40 ++------------------- 2 files changed, 49 insertions(+), 37 deletions(-) create mode 100644 src/allmydata/test/test_humanreadable.py diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py new file mode 100644 index 000000000..ea1d9ce9f --- /dev/null +++ b/src/allmydata/test/test_humanreadable.py @@ -0,0 +1,46 @@ +"""Tests for allmydata.util.humanreadable.""" + +from twisted.trial import unittest + +from allmydata.util import humanreadable + + + + +def foo(): pass # keep the line number constant + + +class NoArgumentException(Exception): + def __init__(self): + pass + +class HumanReadable(unittest.TestCase): + def test_repr(self): + hr = humanreadable.hr + self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(self.test_repr), + ">") + self.failUnlessEqual(hr(long(1)), "1") + self.failUnlessEqual(hr(10**40), + "100000000000000000...000000000000000000") + self.failUnlessEqual(hr(self), "") + self.failUnlessEqual(hr([1,2]), "[1, 2]") + self.failUnlessEqual(hr({1:2}), "{1:2}") + try: + raise ValueError + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "ValueError()") # python-2.5 + try: + raise ValueError("oops") + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "ValueError('oops',)") # python-2.5 + try: + raise NoArgumentException + except Exception as e: + self.failUnless( + hr(e) == "" # python-2.4 + or hr(e) == "NoArgumentException()") # python-2.5 diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 3bddc8bcc..670304a8b 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -1,8 +1,5 @@ from __future__ import print_function - -def foo(): pass # keep the line number constant - import binascii import six import hashlib @@ -17,7 +14,7 @@ from twisted.internet import defer, reactor from twisted.python.failure import Failure from twisted.python import log -from allmydata.util import base32, idlib, humanreadable, mathutil, hashutil +from allmydata.util import base32, idlib, mathutil, hashutil from allmydata.util import assertutil, fileutil, deferredutil, abbreviate from allmydata.util import limiter, time_format, pollmixin from allmydata.util import statistics, dictutil, pipeline, yamlutil @@ -57,44 +54,13 @@ class Base32(unittest.TestCase): self.failUnlessEqual(base32.a2b("ci2a"), "\x12\x34") self.failUnlessRaises(AssertionError, base32.a2b, "b0gus") + class IDLib(unittest.TestCase): def test_nodeid_b2a(self): self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) -class NoArgumentException(Exception): - def __init__(self): - pass -class HumanReadable(unittest.TestCase): - def test_repr(self): - hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") - self.failUnlessEqual(hr(self.test_repr), - ">") - self.failUnlessEqual(hr(long(1)), "1") - self.failUnlessEqual(hr(10**40), - "100000000000000000...000000000000000000") - self.failUnlessEqual(hr(self), "") - self.failUnlessEqual(hr([1,2]), "[1, 2]") - self.failUnlessEqual(hr({1:2}), "{1:2}") - try: - raise ValueError - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "ValueError()") # python-2.5 - try: - raise ValueError("oops") - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "ValueError('oops',)") # python-2.5 - try: - raise NoArgumentException - except Exception as e: - self.failUnless( - hr(e) == "" # python-2.4 - or hr(e) == "NoArgumentException()") # python-2.5 +class Abbreviate(unittest.TestCase): def test_abbrev_time_1s(self): diff = timedelta(seconds=1) From 99b62775e901cd705538df598f1d27b7aa524414 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 13:42:30 -0400 Subject: [PATCH 110/134] Initial list of ported modules. --- src/allmydata/util/_python3.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 src/allmydata/util/_python3.py diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py new file mode 100644 index 000000000..c91e8c400 --- /dev/null +++ b/src/allmydata/util/_python3.py @@ -0,0 +1,3 @@ +"""Track the port to Python 3.""" + +PORTED_MODULES = [] From c207a0c9329faef523e1c76eca1c7b27e2198c19 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:10:38 -0400 Subject: [PATCH 111/134] Port test_humanreadable.py to Python 3. --- setup.py | 3 +++ src/allmydata/test/test_humanreadable.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index caa37f16e..f773652af 100644 --- a/setup.py +++ b/setup.py @@ -119,6 +119,9 @@ install_requires = [ # WebSocket library for twisted and asyncio "autobahn >= 19.5.2", + + # Support for Python 3 transition + "future >= 0.18.2", ] setup_requires = [ diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index ea1d9ce9f..7df6e2cf3 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -1,12 +1,17 @@ """Tests for allmydata.util.humanreadable.""" +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from future import standard_library +standard_library.install_aliases() +from builtins import * from twisted.trial import unittest from allmydata.util import humanreadable - - def foo(): pass # keep the line number constant @@ -17,7 +22,7 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(foo), "") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") @@ -37,7 +42,9 @@ class HumanReadable(unittest.TestCase): except Exception as e: self.failUnless( hr(e) == "" # python-2.4 - or hr(e) == "ValueError('oops',)") # python-2.5 + or hr(e) == "ValueError('oops',)" # python-2.5 + or hr(e) == "ValueError(u'oops',)" # python 2 during py3 transition + ) try: raise NoArgumentException except Exception as e: From 70a029fb81aa79a2082d30fe81147ab5fd04c6fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:26:35 -0400 Subject: [PATCH 112/134] Tests pass on both Python 2 and Python 3. --- setup.py | 4 +++- src/allmydata/test/test_humanreadable.py | 10 ++++---- src/allmydata/util/humanreadable.py | 29 +++++++++++++++--------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index f773652af..9b0436035 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,9 @@ install_requires = [ # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely - "foolscap == 0.13.1", + # * foolscap >= 20.4 is necessary for Python 3 + "foolscap == 0.13.1; ; python_version < '3.0'", + "foolscap >= 20.4.0; ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 7df6e2cf3..2cb66cdaf 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -6,7 +6,7 @@ from __future__ import print_function from future import standard_library standard_library.install_aliases() -from builtins import * +from past.builtins import long from twisted.trial import unittest from allmydata.util import humanreadable @@ -26,8 +26,9 @@ class HumanReadable(unittest.TestCase): self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") - self.failUnlessEqual(hr(10**40), - "100000000000000000...000000000000000000") + self.assertIn(hr(10**40), + ["100000000000000000...000000000000000000", + "100000000000000000...0000000000000000000"]) self.failUnlessEqual(hr(self), "") self.failUnlessEqual(hr([1,2]), "[1, 2]") self.failUnlessEqual(hr({1:2}), "{1:2}") @@ -50,4 +51,5 @@ class HumanReadable(unittest.TestCase): except Exception as e: self.failUnless( hr(e) == "" # python-2.4 - or hr(e) == "NoArgumentException()") # python-2.5 + or hr(e) == "NoArgumentException()" # python-2.5 + or hr(e) == "", hr(e)) # python-3 diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 4d97effe9..8ef15b346 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -1,5 +1,12 @@ -import exceptions, os -from repr import Repr +from __future__ import division +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals +from future import standard_library +standard_library.install_aliases() + +import os +from reprlib import Repr class BetterRepr(Repr, object): def __init__(self): @@ -14,21 +21,21 @@ class BetterRepr(Repr, object): self.maxother = 300 def repr_function(self, obj, level): - if hasattr(obj, 'func_code'): - return '<' + obj.func_name + '() at ' + os.path.basename(obj.func_code.co_filename) + ':' + str(obj.func_code.co_firstlineno) + '>' + if hasattr(obj, '__code__'): + return '<' + obj.__name__ + '() at ' + os.path.basename(obj.__code__.co_filename) + ':' + str(obj.__code__.co_firstlineno) + '>' else: - return '<' + obj.func_name + '() at (builtin)' + return '<' + obj.__name__ + '() at (builtin)' def repr_instance_method(self, obj, level): - if hasattr(obj, 'func_code'): - return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at ' + os.path.basename(obj.im_func.func_code.co_filename) + ':' + str(obj.im_func.func_code.co_firstlineno) + '>' + if hasattr(obj, '__code__'): + return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at ' + os.path.basename(obj.__func__.__code__.co_filename) + ':' + str(obj.__func__.__code__.co_firstlineno) + '>' else: - return '<' + obj.im_class.__name__ + '.' + obj.im_func.__name__ + '() at (builtin)' + return '<' + obj.__self__.__class__.__name__ + '.' + obj.__func__.__name__ + '() at (builtin)' def repr_long(self, obj, level): s = repr(obj) # XXX Hope this isn't too slow... if len(s) > self.maxlong: - i = max(0, (self.maxlong-3)/2) + i = max(0, (self.maxlong-3) // 2) j = max(0, self.maxlong-3-i) s = s[:i] + '...' + s[len(s)-j:] if s[-1] == 'L': @@ -43,7 +50,7 @@ class BetterRepr(Repr, object): on it. If it is an instance of list call self.repr_list() on it. Else call Repr.repr_instance(). """ - if isinstance(obj, exceptions.Exception): + if isinstance(obj, Exception): # Don't cut down exception strings so much. tms = self.maxstring self.maxstring = max(512, tms * 4) @@ -91,7 +98,7 @@ class BetterRepr(Repr, object): if level <= 0: return '{...}' s = '' n = len(obj) - items = obj.items()[:min(n, self.maxdict)] + items = list(obj.items())[:min(n, self.maxdict)] items.sort() for key, val in items: entry = self.repr1(key, level-1) + ':' + self.repr1(val, level-1) From ad7f44db99c856f502ae1f25f0545572111fdcf9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:27:43 -0400 Subject: [PATCH 113/134] Listed ported modules. --- src/allmydata/util/_python3.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index c91e8c400..1dbbb4a1a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,3 +1,10 @@ """Track the port to Python 3.""" -PORTED_MODULES = [] +# Keep these sorted alphabetically, to reduce merge conflicts: +PORTED_MODULES = [ + "allmydata.util.humanreadable", +] + +PORTED_TEST_MODULES = [ + "allmydata.test.test_humanreadable", +] From b77807c558b455a5a7077a1e968b54d60a3bf0c4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:29:15 -0400 Subject: [PATCH 114/134] News entry. --- newsfragments/3324.other | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3324.other diff --git a/newsfragments/3324.other b/newsfragments/3324.other new file mode 100644 index 000000000..e69de29bb From 12691eb597d25eed39ae7edc727a25bf789c3a0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Jul 2020 14:31:33 -0400 Subject: [PATCH 115/134] Only install module mappings once. --- src/allmydata/__init__.py | 5 +++++ src/allmydata/test/test_humanreadable.py | 4 ++-- src/allmydata/util/humanreadable.py | 2 -- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 915b4c013..bbfcb89d4 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -37,3 +37,8 @@ __appname__ = "tahoe-lafs" # in the "application" part of the Tahoe versioning scheme: # https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Versioning __full_version__ = __appname__ + '/' + str(__version__) + + +# Install Python 3 module locations in Python 2: +from future import standard_library +standard_library.install_aliases() diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 2cb66cdaf..94fe07504 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -4,14 +4,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from future import standard_library -standard_library.install_aliases() from past.builtins import long + from twisted.trial import unittest from allmydata.util import humanreadable + def foo(): pass # keep the line number constant diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 8ef15b346..696fc72bb 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -2,8 +2,6 @@ from __future__ import division from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals -from future import standard_library -standard_library.install_aliases() import os from reprlib import Repr From 83182e1d381510e3000ffd0ece8ed2379dfe0144 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:15:13 -0400 Subject: [PATCH 116/134] Remove extraneous semicolon. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9b0436035..fd8d7bcff 100644 --- a/setup.py +++ b/setup.py @@ -55,8 +55,8 @@ install_requires = [ # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs # * foolscap 0.13.2 drops i2p support completely # * foolscap >= 20.4 is necessary for Python 3 - "foolscap == 0.13.1; ; python_version < '3.0'", - "foolscap >= 20.4.0; ; python_version > '3.0'", + "foolscap == 0.13.1 ; python_version < '3.0'", + "foolscap >= 20.4.0 ; python_version > '3.0'", # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that # Twisted[conch] also depends on cryptography and Twisted[tls] From 140cb80ff7be474dcbf5f3db4521ca0ee131ec4e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:15:57 -0400 Subject: [PATCH 117/134] Clarify comment. --- src/allmydata/test/test_humanreadable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 94fe07504..6d90b562e 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -12,7 +12,7 @@ from allmydata.util import humanreadable -def foo(): pass # keep the line number constant +def foo(): pass # FYI foo()'s line number is used in the test below class NoArgumentException(Exception): From 83b9bbf7433825f2f2e4390f51d3dedfd6ec5752 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:28:36 -0400 Subject: [PATCH 118/134] Switch definitely to _python3.py for listing modules, but also note if a module has been ported in its docstring. --- src/allmydata/ported-modules.txt | 1 - src/allmydata/test/test_humanreadable.py | 7 ++++++- src/allmydata/util/_python3.py | 1 + src/allmydata/util/humanreadable.py | 6 ++++++ src/allmydata/util/namespace.py | 3 +++ 5 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 src/allmydata/ported-modules.txt diff --git a/src/allmydata/ported-modules.txt b/src/allmydata/ported-modules.txt deleted file mode 100644 index cd150cbfb..000000000 --- a/src/allmydata/ported-modules.txt +++ /dev/null @@ -1 +0,0 @@ -allmydata.util.namespace diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index 6d90b562e..cbe478daf 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -1,4 +1,9 @@ -"""Tests for allmydata.util.humanreadable.""" +""" +Tests for allmydata.util.humanreadable. + +This module has been ported to Python 3. +""" + from __future__ import unicode_literals from __future__ import absolute_import from __future__ import division diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1dbbb4a1a..4f023ade7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -3,6 +3,7 @@ # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.humanreadable", + "allmydata.util.namespace", ] PORTED_TEST_MODULES = [ diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 696fc72bb..edddcdf4b 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -1,3 +1,9 @@ +""" +Utilities for turning objects into human-readable strings. + +This module has been ported to Python 3. +""" + from __future__ import division from __future__ import absolute_import from __future__ import print_function diff --git a/src/allmydata/util/namespace.py b/src/allmydata/util/namespace.py index c386b7376..247827c47 100644 --- a/src/allmydata/util/namespace.py +++ b/src/allmydata/util/namespace.py @@ -1,3 +1,6 @@ +""" +This module has been ported to Python 3. +""" class Namespace(object): pass From 6d4ce3b7ad5ddc48c1a883f7855bc4d773a0bf62 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Jul 2020 13:44:47 -0400 Subject: [PATCH 119/134] Additional hidden imports due to use of Future. --- pyinstaller.spec | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 6d1bd21f2..875629c13 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -27,15 +27,29 @@ added_files = [ ('src/allmydata/web/static/img/*.png', 'allmydata/web/static/img')] hidden_imports = [ + '__builtin__', 'allmydata.client', 'allmydata.introducer', 'allmydata.stats', + 'base64', 'cffi', + 'collections', + 'commands', 'Crypto', + 'functools', + 'future.backports.misc', + 'itertools', + 'math', 'packaging.specifiers', + 're', + 'reprlib', 'six.moves.html_parser', + 'subprocess', + 'UserDict', + 'UserList', + 'UserString', 'yaml', - 'zfec' + 'zfec', ] a = Analysis( From 8a85bf783645e7510087001b67a3ce06d0c7fc7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 6 Jul 2020 10:41:38 -0400 Subject: [PATCH 120/134] add a contributor covenant badge --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d53fa0104..274cee41d 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,8 @@ function correctly, preserving your privacy and security. For full documentation, please see http://tahoe-lafs.readthedocs.io/en/latest/ . -|readthedocs| |travis| |circleci| |codecov| +|Contributor Covenant| |readthedocs| |travis| |circleci| |codecov| + INSTALLING ========== @@ -105,3 +106,5 @@ slides. .. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md From 906d6ac29151c4e5f69c3f70b7e6aa03af7bc4c9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 13:50:06 -0400 Subject: [PATCH 121/134] Switch to flake8 as a better way of running pyflakes. --- Makefile | 2 +- newsfragments/3329.other | 0 setup.cfg | 5 +++++ setup.py | 3 ++- tox.ini | 2 +- 5 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3329.other diff --git a/Makefile b/Makefile index 6e0009ddf..6dd87409e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ default: PYTHON=python export PYTHON -PYFLAKES=pyflakes +PYFLAKES=flake8 export PYFLAKES SOURCES=src/allmydata static misc setup.py diff --git a/newsfragments/3329.other b/newsfragments/3329.other new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index de40c9e23..c2d224225 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,8 @@ install = update_version install develop = update_version develop bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel + +[flake8] +# For now, only use pyflakes errors; flake8 is still helpful because it allows +# ignoring specific errors/warnings when needed. +select = F \ No newline at end of file diff --git a/setup.py b/setup.py index caa37f16e..0d43294cc 100644 --- a/setup.py +++ b/setup.py @@ -353,11 +353,12 @@ setup(name="tahoe-lafs", # also set in __init__.py # discussion. ':sys_platform=="win32"': ["pywin32 != 226"], "test": [ + "flake8", # Pin a specific pyflakes so we don't have different folks # disagreeing on what is or is not a lint issue. We can bump # this version from time to time, but we will do it # intentionally. - "pyflakes == 2.1.0", + "pyflakes == 2.2.0", # coverage 5.0 breaks the integration tests in some opaque way. # This probably needs to be addressed in a more permanent way # eventually... diff --git a/tox.ini b/tox.ini index 3134a0d9f..44b583baa 100644 --- a/tox.ini +++ b/tox.ini @@ -75,7 +75,7 @@ commands = whitelist_externals = /bin/mv commands = - pyflakes src static misc setup.py + flake8 src static misc setup.py python misc/coding_tools/check-umids.py src python misc/coding_tools/check-debugging.py python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py From b99033410431b3434d0e6f4614c127b118d36584 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 14:00:02 -0400 Subject: [PATCH 122/134] Add the builtins import to make Python 2 builtins match Python 3's. --- src/allmydata/test/test_humanreadable.py | 6 +++++- src/allmydata/util/humanreadable.py | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index cbe478daf..ec6d3f666 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -9,6 +9,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from past.builtins import long from twisted.trial import unittest @@ -27,7 +31,7 @@ class NoArgumentException(Exception): class HumanReadable(unittest.TestCase): def test_repr(self): hr = humanreadable.hr - self.failUnlessEqual(hr(foo), "") + self.failUnlessEqual(hr(foo), "") self.failUnlessEqual(hr(self.test_repr), ">") self.failUnlessEqual(hr(long(1)), "1") diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index edddcdf4b..8c7079c3a 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -9,6 +9,10 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + import os from reprlib import Repr From f548593f374bcf499c358faa1f56c6d85a4acf9e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 6 Jul 2020 14:18:18 -0400 Subject: [PATCH 123/134] Fix test_python3, and mark these two modules as being ported as well. --- src/allmydata/test/test_python3.py | 40 +++++++++++++++++++++++------- src/allmydata/util/_python3.py | 17 ++++++++++++- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 8a25af889..294323b68 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -1,6 +1,18 @@ """ Tests related to the Python 3 porting effort itself. + +This module has been ported to Python 3. """ +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if False:#PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from unittest import expectedFailure from pkg_resources import ( resource_stream, @@ -13,11 +25,18 @@ from twisted.trial.unittest import ( SynchronousTestCase, ) +from allmydata.util._python3 import PORTED_MODULES, PORTED_TEST_MODULES + class Python3PortingEffortTests(SynchronousTestCase): + def test_finished_porting(self): """ Tahoe-LAFS has been ported to Python 3. + + Once + https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 + is completed this test should pass (and can be deleted!). """ tahoe_lafs_module_names = set(all_module_names("allmydata")) ported_names = set(ported_module_names()) @@ -31,7 +50,10 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed" + if PY2: + test_finished_porting.skip = "For some reason todo isn't working on Python 2 now" + else: + test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed" def test_ported_modules_exist(self): """ @@ -70,18 +92,18 @@ def all_module_names(toplevel): """ allmydata = getModule(toplevel) for module in allmydata.walkModules(): - yield module.name.decode("utf-8") + name = module.name + if PY2: + name = name.decode("utf-8") + yield name def ported_module_names(): """ - :return list[unicode]: A ``set`` of ``unicode`` giving the names of + :return list[unicode]: A ``list`` of ``unicode`` giving the names of Tahoe-LAFS modules which have been ported to Python 3. """ - return resource_stream( - "allmydata", - u"ported-modules.txt", - ).read().splitlines() + return PORTED_MODULES + PORTED_TEST_MODULES def unported_report(tahoe_lafs_module_names, ported_names): @@ -100,8 +122,8 @@ def count_lines(module_name): try: source = module.filePath.getContent() except Exception as e: - print(module_name, e) + print((module_name, e)) return 0 lines = source.splitlines() - nonblank = filter(None, lines) + nonblank = [_f for _f in lines if _f] return len(nonblank) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4f023ade7..fd679e014 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -1,11 +1,26 @@ -"""Track the port to Python 3.""" +""" +Track the port to Python 3. + +This module has been ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata.util.humanreadable", "allmydata.util.namespace", + "allmydata.util._python3", ] PORTED_TEST_MODULES = [ "allmydata.test.test_humanreadable", + "allmydata.test.test_python3", ] From b9737febd0ee1f497bc5e6fc92acb342fdd52013 Mon Sep 17 00:00:00 2001 From: meejah Date: Mon, 6 Jul 2020 13:49:07 -0600 Subject: [PATCH 124/134] add meejah to code-of-conduct --- docs/CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md index 868f59c6a..3b14f2d70 100644 --- a/docs/CODE_OF_CONDUCT.md +++ b/docs/CODE_OF_CONDUCT.md @@ -44,6 +44,7 @@ incident. The following community members have made themselves available for conduct issues: - Jean-Paul Calderone (jean-paul at leastauthority dot com) +- meejah (meejah at meejah dot ca) This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.3.0, available at From 0ed8a7709e4ff1d78d0ba19db4c537c824f1aa41 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:18:11 -0400 Subject: [PATCH 125/134] Perhaps this is the preferred formatting? --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 274cee41d..082925670 100644 --- a/README.rst +++ b/README.rst @@ -107,4 +107,5 @@ slides. :alt: test coverage percentage :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master -.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg :target: docs/CODE_OF_CONDUCT.md +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :target: docs/CODE_OF_CONDUCT.md From f08e2ba832f829c5469b87752235b0161c54902b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Jul 2020 09:19:30 -0400 Subject: [PATCH 126/134] add some alt text --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 082925670..d3b089186 100644 --- a/README.rst +++ b/README.rst @@ -108,4 +108,5 @@ slides. :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg + :alt: code of conduct :target: docs/CODE_OF_CONDUCT.md From e6268f3dd5a762d3e6b6abf1327f6046d321875c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:13:45 -0400 Subject: [PATCH 127/134] Add additional dependencies. --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index cc0eea479..3b54519f1 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib +, html5lib, future }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; From 238b17098e1fff93cdcd2f61abdaf1b7bafc646a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:28:11 -0400 Subject: [PATCH 128/134] Add future in another place. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 3b54519f1..d82c97db7 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,6 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools + future ]; checkInputs = with python.pkgs; [ From 3d0e4df5cec43570d0b5cb672db14b945efd381c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 7 Jul 2020 11:30:01 -0400 Subject: [PATCH 129/134] Lint fixes. --- src/allmydata/test/test_python3.py | 8 +--- src/allmydata/test/test_util.py | 74 ++++++++++++++---------------- 2 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 294323b68..1326c8282 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -9,15 +9,9 @@ from __future__ import division from __future__ import print_function from future.utils import PY2 -if False:#PY2: +if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 -from unittest import expectedFailure - -from pkg_resources import ( - resource_stream, -) - from twisted.python.modules import ( getModule, ) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 670304a8b..65ba308d9 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -60,45 +60,6 @@ class IDLib(unittest.TestCase): self.failUnlessEqual(idlib.nodeid_b2a("\x00"*20), "a"*32) -class Abbreviate(unittest.TestCase): - - def test_abbrev_time_1s(self): - diff = timedelta(seconds=1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('1 second ago', s) - - def test_abbrev_time_25s(self): - diff = timedelta(seconds=25) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('25 seconds ago', s) - - def test_abbrev_time_future_5_minutes(self): - diff = timedelta(minutes=-5) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 minutes in the future', s) - - def test_abbrev_time_hours(self): - diff = timedelta(hours=4) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('4 hours ago', s) - - def test_abbrev_time_day(self): - diff = timedelta(hours=49) # must be more than 2 days - s = abbreviate.abbreviate_time(diff) - self.assertEqual('2 days ago', s) - - def test_abbrev_time_month(self): - diff = timedelta(days=91) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('3 months ago', s) - - def test_abbrev_time_year(self): - diff = timedelta(weeks=(5 * 52) + 1) - s = abbreviate.abbreviate_time(diff) - self.assertEqual('5 years ago', s) - - - class MyList(list): pass @@ -950,6 +911,41 @@ class HashUtilTests(unittest.TestCase): ) class Abbreviate(unittest.TestCase): + def test_abbrev_time_1s(self): + diff = timedelta(seconds=1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('1 second ago', s) + + def test_abbrev_time_25s(self): + diff = timedelta(seconds=25) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('25 seconds ago', s) + + def test_abbrev_time_future_5_minutes(self): + diff = timedelta(minutes=-5) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 minutes in the future', s) + + def test_abbrev_time_hours(self): + diff = timedelta(hours=4) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('4 hours ago', s) + + def test_abbrev_time_day(self): + diff = timedelta(hours=49) # must be more than 2 days + s = abbreviate.abbreviate_time(diff) + self.assertEqual('2 days ago', s) + + def test_abbrev_time_month(self): + diff = timedelta(days=91) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('3 months ago', s) + + def test_abbrev_time_year(self): + diff = timedelta(weeks=(5 * 52) + 1) + s = abbreviate.abbreviate_time(diff) + self.assertEqual('5 years ago', s) + def test_time(self): a = abbreviate.abbreviate_time self.failUnlessEqual(a(None), "unknown") From 3d80a6c17e13f3eb51564423b58df2a25c2adc6c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 10:11:28 -0400 Subject: [PATCH 130/134] New packages so we can get future 0.18.2. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..ffb0eb4d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,7 +329,7 @@ jobs: - image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03-small.tar.gz" steps: - "checkout" From 1cbf7981276a49b5034eb664141b6c7a472341c5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:07 -0400 Subject: [PATCH 131/134] Package Future 0.18.2 for us --- nix/future.nix | 35 +++++++++++++++++++++++++++++++++++ nix/overlays.nix | 5 +++++ nix/tahoe-lafs.nix | 1 + 3 files changed, 41 insertions(+) create mode 100644 nix/future.nix diff --git a/nix/future.nix b/nix/future.nix new file mode 100644 index 000000000..814b7c1b5 --- /dev/null +++ b/nix/future.nix @@ -0,0 +1,35 @@ +{ lib +, buildPythonPackage +, fetchPypi +}: + +buildPythonPackage rec { + pname = "future"; + version = "0.18.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi"; + }; + + doCheck = false; + + meta = { + description = "Clean single-source support for Python 3 and 2"; + longDescription = '' + python-future is the missing compatibility layer between Python 2 and + Python 3. It allows you to use a single, clean Python 3.x-compatible + codebase to support both Python 2 and Python 3 with minimal overhead. + + It provides future and past packages with backports and forward ports + of features from Python 3 and 2. It also comes with futurize and + pasteurize, customized 2to3-based scripts that helps you to convert + either Py2 or Py3 code easily to support both Python 2 and 3 in a + single clean Py3-style codebase, module by module. + ''; + homepage = https://python-future.org; + downloadPage = https://github.com/PythonCharmers/python-future/releases; + license = with lib.licenses; [ mit ]; + maintainers = with lib.maintainers; [ prikhi ]; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 2a9fe8e5e..7f19a8f1a 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -10,6 +10,11 @@ self: super: { # NixOS autobahn package has trollius as a dependency, although # it is optional. Trollius is unmaintained and fails on CI. autobahn = python-super.callPackage ./autobahn.nix { }; + + # Porting to Python 3 is greatly aided by the future package. A + # slightly newer version than appears in nixos 19.09 is helpful. + future = python-super.callPackage ./future.nix { }; + }; }; } diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index cc0eea479..ae5c05f70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,6 +50,7 @@ python.pkgs.buildPythonPackage rec { setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography setuptools + future ]; checkInputs = with python.pkgs; [ From 7e16148f83cabb0507560ef45c60b9d84b38de8a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Jul 2020 13:20:42 -0400 Subject: [PATCH 132/134] news fragment --- newsfragments/3338.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3338.minor diff --git a/newsfragments/3338.minor b/newsfragments/3338.minor new file mode 100644 index 000000000..e69de29bb From 98c8b2fd00b3823988d4d253ad5600ed56bd80f7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 14:53:38 -0400 Subject: [PATCH 133/134] Revert changes obsoleted by #3338. --- .circleci/config.yml | 2 +- nix/tahoe-lafs.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffb0eb4d8..d0f596fc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -329,7 +329,7 @@ jobs: - image: "nixorg/nix:circleci" environment: - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-20.03-small.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" steps: - "checkout" diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index d82c97db7..ae5c05f70 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -4,7 +4,7 @@ , setuptools, setuptoolsTrial, pyasn1, zope_interface , service-identity, pyyaml, magic-wormhole, treq, appdirs , beautifulsoup4, eliot, autobahn, cryptography -, html5lib, future +, html5lib }: python.pkgs.buildPythonPackage rec { version = "1.14.0.dev"; From 28f219ad3e036a8db35a81a587d443671f50f16c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Jul 2020 16:46:54 -0400 Subject: [PATCH 134/134] Update to version that supports _python3.py variant of port list. --- misc/python3/tahoe-depgraph.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/misc/python3/tahoe-depgraph.py b/misc/python3/tahoe-depgraph.py index 1bfe89c73..0f624fa70 100644 --- a/misc/python3/tahoe-depgraph.py +++ b/misc/python3/tahoe-depgraph.py @@ -60,7 +60,8 @@ class mymf(modulefinder.ModuleFinder): self._depgraph[last_caller.__name__].add(fqname) return r - def load_module(self, fqname, fp, pathname, (suffix, mode, type)): + def load_module(self, fqname, fp, pathname, additional_info): + (suffix, mode, type) = additional_info r = modulefinder.ModuleFinder.load_module( self, fqname, fp, pathname, (suffix, mode, type)) if r is not None: @@ -71,7 +72,7 @@ class mymf(modulefinder.ModuleFinder): return { 'depgraph': { name: dict.fromkeys(deps, 1) - for name, deps in self._depgraph.iteritems()}, + for name, deps in self._depgraph.items()}, 'types': self._types, } @@ -101,20 +102,25 @@ def main(target): filepath = path moduleNames.append(reflect.filenameToModuleName(filepath)) - with tempfile.NamedTemporaryFile() as tmpfile: + with tempfile.NamedTemporaryFile("w") as tmpfile: for moduleName in moduleNames: tmpfile.write('import %s\n' % moduleName) tmpfile.flush() mf.run_script(tmpfile.name) - with open('tahoe-deps.json', 'wb') as outfile: + with open('tahoe-deps.json', 'w') as outfile: json_dump(mf.as_json(), outfile) outfile.write('\n') - ported_modules_path = os.path.join(target, "src", "allmydata", "ported-modules.txt") - with open(ported_modules_path) as ported_modules: - port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported") - with open('tahoe-ported.json', 'wb') as outfile: + ported_modules_path = os.path.join(target, "src", "allmydata", "util", "_python3.py") + with open(ported_modules_path) as f: + ported_modules = {} + exec(f.read(), ported_modules, ported_modules) + port_status = dict.fromkeys( + ported_modules["PORTED_MODULES"] + ported_modules["PORTED_TEST_MODULES"], + "ported" + ) + with open('tahoe-ported.json', 'w') as outfile: json_dump(port_status, outfile) outfile.write('\n')